diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2016-07-23 20:41:05 +0000 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2016-07-23 20:41:05 +0000 |
| commit | 01095a5d43bbfde13731688ddcf6048ebb8b7721 (patch) | |
| tree | 4def12e759965de927d963ac65840d663ef9d1ea /lib/IR | |
| parent | f0f4822ed4b66e3579e92a89f368f8fb860e218e (diff) | |
| download | src-test-01095a5d43bbfde13731688ddcf6048ebb8b7721.tar.gz src-test-01095a5d43bbfde13731688ddcf6048ebb8b7721.zip | |
Notes
Diffstat (limited to 'lib/IR')
54 files changed, 5659 insertions, 3060 deletions
diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp index 0ce44e105cc39..9b2399dd880c3 100644 --- a/lib/IR/AsmWriter.cpp +++ b/lib/IR/AsmWriter.cpp @@ -102,6 +102,11 @@ static OrderMap orderModule(const Module *M) { orderValue(A.getAliasee(), OM); orderValue(&A, OM); } + for (const GlobalIFunc &I : M->ifuncs()) { + if (!isa<GlobalValue>(I.getResolver())) + orderValue(I.getResolver(), OM); + orderValue(&I, OM); + } for (const Function &F : *M) { for (const Use &U : F.operands()) if (!isa<GlobalValue>(U.get())) @@ -249,11 +254,15 @@ static UseListOrderStack predictUseListOrder(const Module *M) { predictValueUseListOrder(&F, nullptr, OM, Stack); for (const GlobalAlias &A : M->aliases()) predictValueUseListOrder(&A, nullptr, OM, Stack); + for (const GlobalIFunc &I : M->ifuncs()) + predictValueUseListOrder(&I, nullptr, OM, Stack); for (const GlobalVariable &G : M->globals()) if (G.hasInitializer()) predictValueUseListOrder(G.getInitializer(), nullptr, OM, Stack); for (const GlobalAlias &A : M->aliases()) predictValueUseListOrder(A.getAliasee(), nullptr, OM, Stack); + for (const GlobalIFunc &I : M->ifuncs()) + predictValueUseListOrder(I.getResolver(), nullptr, OM, Stack); for (const Function &F : *M) for (const Use &U : F.operands()) predictValueUseListOrder(U.get(), nullptr, OM, Stack); @@ -307,15 +316,23 @@ static void PrintCallingConv(unsigned cc, raw_ostream &Out) { case CallingConv::ARM_AAPCS: Out << "arm_aapcscc"; break; case CallingConv::ARM_AAPCS_VFP: Out << "arm_aapcs_vfpcc"; break; case CallingConv::MSP430_INTR: Out << "msp430_intrcc"; break; + case CallingConv::AVR_INTR: Out << "avr_intrcc "; break; + case CallingConv::AVR_SIGNAL: Out << "avr_signalcc "; break; case CallingConv::PTX_Kernel: Out << "ptx_kernel"; break; case CallingConv::PTX_Device: Out << "ptx_device"; break; case CallingConv::X86_64_SysV: Out << "x86_64_sysvcc"; break; case CallingConv::X86_64_Win64: Out << "x86_64_win64cc"; break; case CallingConv::SPIR_FUNC: Out << "spir_func"; break; case CallingConv::SPIR_KERNEL: Out << "spir_kernel"; break; + case CallingConv::Swift: Out << "swiftcc"; break; case CallingConv::X86_INTR: Out << "x86_intrcc"; break; case CallingConv::HHVM: Out << "hhvmcc"; break; case CallingConv::HHVM_C: Out << "hhvm_ccc"; break; + case CallingConv::AMDGPU_VS: Out << "amdgpu_vs"; break; + case CallingConv::AMDGPU_GS: Out << "amdgpu_gs"; break; + case CallingConv::AMDGPU_PS: Out << "amdgpu_ps"; break; + case CallingConv::AMDGPU_CS: Out << "amdgpu_cs"; break; + case CallingConv::AMDGPU_KERNEL: Out << "amdgpu_kernel"; break; } } @@ -664,6 +681,9 @@ private: /// Add all of the functions arguments, basic blocks, and instructions. void processFunction(); + /// Add the metadata directly attached to a GlobalObject. + void processGlobalObjectMetadata(const GlobalObject &GO); + /// Add all of the metadata from a function. void processFunctionMetadata(const Function &F); @@ -681,14 +701,25 @@ ModuleSlotTracker::ModuleSlotTracker(SlotTracker &Machine, const Module *M, ModuleSlotTracker::ModuleSlotTracker(const Module *M, bool ShouldInitializeAllMetadata) - : MachineStorage(M ? new SlotTracker(M, ShouldInitializeAllMetadata) - : nullptr), - M(M), Machine(MachineStorage.get()) {} + : ShouldCreateStorage(M), + ShouldInitializeAllMetadata(ShouldInitializeAllMetadata), M(M) {} ModuleSlotTracker::~ModuleSlotTracker() {} +SlotTracker *ModuleSlotTracker::getMachine() { + if (!ShouldCreateStorage) + return Machine; + + ShouldCreateStorage = false; + MachineStorage = + llvm::make_unique<SlotTracker>(M, ShouldInitializeAllMetadata); + Machine = MachineStorage.get(); + return Machine; +} + void ModuleSlotTracker::incorporateFunction(const Function &F) { - if (!Machine) + // Using getMachine() may lazily create the slot tracker. + if (!getMachine()) return; // Nothing to do if this is the right function already. @@ -722,6 +753,9 @@ static SlotTracker *createSlotTracker(const Value *V) { if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) return new SlotTracker(GA->getParent()); + if (const GlobalIFunc *GIF = dyn_cast<GlobalIFunc>(V)) + return new SlotTracker(GIF->getParent()); + if (const Function *Func = dyn_cast<Function>(V)) return new SlotTracker(Func); @@ -768,6 +802,7 @@ void SlotTracker::processModule() { for (const GlobalVariable &Var : TheModule->globals()) { if (!Var.hasName()) CreateModuleSlot(&Var); + processGlobalObjectMetadata(Var); } for (const GlobalAlias &A : TheModule->aliases()) { @@ -775,6 +810,11 @@ void SlotTracker::processModule() { CreateModuleSlot(&A); } + for (const GlobalIFunc &I : TheModule->ifuncs()) { + if (!I.hasName()) + CreateModuleSlot(&I); + } + // Add metadata used by named metadata. for (const NamedMDNode &NMD : TheModule->named_metadata()) { for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i) @@ -846,12 +886,15 @@ void SlotTracker::processFunction() { ST_DEBUG("end processFunction!\n"); } -void SlotTracker::processFunctionMetadata(const Function &F) { +void SlotTracker::processGlobalObjectMetadata(const GlobalObject &GO) { SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; - F.getAllMetadata(MDs); + GO.getAllMetadata(MDs); for (auto &MD : MDs) CreateMetadataSlot(MD.second); +} +void SlotTracker::processFunctionMetadata(const Function &F) { + processGlobalObjectMetadata(F); for (auto &BB : F) { for (auto &I : BB) processInstructionMetadata(I); @@ -938,10 +981,11 @@ void SlotTracker::CreateModuleSlot(const GlobalValue *V) { ST_DEBUG(" Inserting value [" << V->getType() << "] = " << V << " slot=" << DestSlot << " ["); - // G = Global, F = Function, A = Alias, o = other + // G = Global, F = Function, A = Alias, I = IFunc, o = other ST_DEBUG((isa<GlobalVariable>(V) ? 'G' : (isa<Function>(V) ? 'F' : - (isa<GlobalAlias>(V) ? 'A' : 'o'))) << "]\n"); + (isa<GlobalAlias>(V) ? 'A' : + (isa<GlobalIFunc>(V) ? 'I' : 'o')))) << "]\n"); } /// CreateSlot - Create a new slot for the specified value if it has no name. @@ -1401,6 +1445,7 @@ struct MDFieldPrinter { template <class IntTy, class Stringifier> void printDwarfEnum(StringRef Name, IntTy Value, Stringifier toString, bool ShouldSkipZero = true); + void printEmissionKind(StringRef Name, DICompileUnit::DebugEmissionKind EK); }; } // end namespace @@ -1481,6 +1526,12 @@ void MDFieldPrinter::printDIFlags(StringRef Name, unsigned Flags) { Out << FlagsFS << Extra; } +void MDFieldPrinter::printEmissionKind(StringRef Name, + DICompileUnit::DebugEmissionKind EK) { + Out << FS << Name << ": " << DICompileUnit::EmissionKindString(EK); +} + + template <class IntTy, class Stringifier> void MDFieldPrinter::printDwarfEnum(StringRef Name, IntTy Value, Stringifier toString, bool ShouldSkipZero) { @@ -1608,6 +1659,7 @@ static void writeDISubroutineType(raw_ostream &Out, const DISubroutineType *N, Out << "!DISubroutineType("; MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); Printer.printDIFlags("flags", N->getFlags()); + Printer.printDwarfEnum("cc", N->getCC(), dwarf::ConventionString); Printer.printMetadata("types", N->getRawTypeArray(), /* ShouldSkipNull */ false); Out << ")"; @@ -1638,11 +1690,9 @@ static void writeDICompileUnit(raw_ostream &Out, const DICompileUnit *N, Printer.printInt("runtimeVersion", N->getRuntimeVersion(), /* ShouldSkipZero */ false); Printer.printString("splitDebugFilename", N->getSplitDebugFilename()); - Printer.printInt("emissionKind", N->getEmissionKind(), - /* ShouldSkipZero */ false); + Printer.printEmissionKind("emissionKind", N->getEmissionKind()); Printer.printMetadata("enums", N->getRawEnumTypes()); Printer.printMetadata("retainedTypes", N->getRawRetainedTypes()); - Printer.printMetadata("subprograms", N->getRawSubprograms()); Printer.printMetadata("globals", N->getRawGlobalVariables()); Printer.printMetadata("imports", N->getRawImportedEntities()); Printer.printMetadata("macros", N->getRawMacros()); @@ -1667,9 +1717,13 @@ static void writeDISubprogram(raw_ostream &Out, const DISubprogram *N, Printer.printMetadata("containingType", N->getRawContainingType()); Printer.printDwarfEnum("virtuality", N->getVirtuality(), dwarf::VirtualityString); - Printer.printInt("virtualIndex", N->getVirtualIndex()); + if (N->getVirtuality() != dwarf::DW_VIRTUALITY_none || + N->getVirtualIndex() != 0) + Printer.printInt("virtualIndex", N->getVirtualIndex(), false); + Printer.printInt("thisAdjustment", N->getThisAdjustment()); Printer.printDIFlags("flags", N->getFlags()); Printer.printBool("isOptimized", N->isOptimized()); + Printer.printMetadata("unit", N->getRawUnit()); Printer.printMetadata("templateParams", N->getRawTemplateParams()); Printer.printMetadata("declaration", N->getRawDeclaration()); Printer.printMetadata("variables", N->getRawVariables()); @@ -2037,7 +2091,7 @@ public: void printTypeIdentities(); void printGlobal(const GlobalVariable *GV); - void printAlias(const GlobalAlias *GV); + void printIndirectSymbol(const GlobalIndirectSymbol *GIS); void printComdat(const Comdat *C); void printFunction(const Function *F); void printArgument(const Argument *FA, AttributeSet Attrs, unsigned Idx); @@ -2073,11 +2127,8 @@ AssemblyWriter::AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac, if (!TheModule) return; TypePrinter.incorporateTypes(*TheModule); - for (const Function &F : *TheModule) - if (const Comdat *C = F.getComdat()) - Comdats.insert(C); - for (const GlobalVariable &GV : TheModule->globals()) - if (const Comdat *C = GV.getComdat()) + for (const GlobalObject &GO : TheModule->global_objects()) + if (const Comdat *C = GO.getComdat()) Comdats.insert(C); } @@ -2095,7 +2146,7 @@ void AssemblyWriter::writeOperand(const Value *Operand, bool PrintType) { void AssemblyWriter::writeAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) { - if (Ordering == NotAtomic) + if (Ordering == AtomicOrdering::NotAtomic) return; switch (SynchScope) { @@ -2103,46 +2154,22 @@ void AssemblyWriter::writeAtomic(AtomicOrdering Ordering, case CrossThread: break; } - switch (Ordering) { - default: Out << " <bad ordering " << int(Ordering) << ">"; break; - case Unordered: Out << " unordered"; break; - case Monotonic: Out << " monotonic"; break; - case Acquire: Out << " acquire"; break; - case Release: Out << " release"; break; - case AcquireRelease: Out << " acq_rel"; break; - case SequentiallyConsistent: Out << " seq_cst"; break; - } + Out << " " << toIRString(Ordering); } void AssemblyWriter::writeAtomicCmpXchg(AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) { - assert(SuccessOrdering != NotAtomic && FailureOrdering != NotAtomic); + assert(SuccessOrdering != AtomicOrdering::NotAtomic && + FailureOrdering != AtomicOrdering::NotAtomic); switch (SynchScope) { case SingleThread: Out << " singlethread"; break; case CrossThread: break; } - switch (SuccessOrdering) { - default: Out << " <bad ordering " << int(SuccessOrdering) << ">"; break; - case Unordered: Out << " unordered"; break; - case Monotonic: Out << " monotonic"; break; - case Acquire: Out << " acquire"; break; - case Release: Out << " release"; break; - case AcquireRelease: Out << " acq_rel"; break; - case SequentiallyConsistent: Out << " seq_cst"; break; - } - - switch (FailureOrdering) { - default: Out << " <bad ordering " << int(FailureOrdering) << ">"; break; - case Unordered: Out << " unordered"; break; - case Monotonic: Out << " monotonic"; break; - case Acquire: Out << " acquire"; break; - case Release: Out << " release"; break; - case AcquireRelease: Out << " acq_rel"; break; - case SequentiallyConsistent: Out << " seq_cst"; break; - } + Out << " " << toIRString(SuccessOrdering); + Out << " " << toIRString(FailureOrdering); } void AssemblyWriter::writeParamOperand(const Value *Operand, @@ -2211,6 +2238,12 @@ void AssemblyWriter::printModule(const Module *M) { M->getModuleIdentifier().find('\n') == std::string::npos) Out << "; ModuleID = '" << M->getModuleIdentifier() << "'\n"; + if (!M->getSourceFileName().empty()) { + Out << "source_filename = \""; + PrintEscapedString(M->getSourceFileName(), Out); + Out << "\"\n"; + } + const std::string &DL = M->getDataLayoutStr(); if (!DL.empty()) Out << "target datalayout = \"" << DL << "\"\n"; @@ -2254,7 +2287,12 @@ void AssemblyWriter::printModule(const Module *M) { // Output all aliases. if (!M->alias_empty()) Out << "\n"; for (const GlobalAlias &GA : M->aliases()) - printAlias(&GA); + printIndirectSymbol(&GA); + + // Output all ifuncs. + if (!M->ifunc_empty()) Out << "\n"; + for (const GlobalIFunc &GI : M->ifuncs()) + printIndirectSymbol(&GI); // Output global use-lists. printUseLists(nullptr); @@ -2320,23 +2358,32 @@ void AssemblyWriter::printNamedMDNode(const NamedMDNode *NMD) { Out << "}\n"; } -static void PrintLinkage(GlobalValue::LinkageTypes LT, - formatted_raw_ostream &Out) { +static const char *getLinkagePrintName(GlobalValue::LinkageTypes LT) { switch (LT) { - case GlobalValue::ExternalLinkage: break; - case GlobalValue::PrivateLinkage: Out << "private "; break; - case GlobalValue::InternalLinkage: Out << "internal "; break; - case GlobalValue::LinkOnceAnyLinkage: Out << "linkonce "; break; - case GlobalValue::LinkOnceODRLinkage: Out << "linkonce_odr "; break; - case GlobalValue::WeakAnyLinkage: Out << "weak "; break; - case GlobalValue::WeakODRLinkage: Out << "weak_odr "; break; - case GlobalValue::CommonLinkage: Out << "common "; break; - case GlobalValue::AppendingLinkage: Out << "appending "; break; - case GlobalValue::ExternalWeakLinkage: Out << "extern_weak "; break; + case GlobalValue::ExternalLinkage: + return ""; + case GlobalValue::PrivateLinkage: + return "private "; + case GlobalValue::InternalLinkage: + return "internal "; + case GlobalValue::LinkOnceAnyLinkage: + return "linkonce "; + case GlobalValue::LinkOnceODRLinkage: + return "linkonce_odr "; + case GlobalValue::WeakAnyLinkage: + return "weak "; + case GlobalValue::WeakODRLinkage: + return "weak_odr "; + case GlobalValue::CommonLinkage: + return "common "; + case GlobalValue::AppendingLinkage: + return "appending "; + case GlobalValue::ExternalWeakLinkage: + return "extern_weak "; case GlobalValue::AvailableExternallyLinkage: - Out << "available_externally "; - break; + return "available_externally "; } + llvm_unreachable("invalid linkage"); } static void PrintVisibility(GlobalValue::VisibilityTypes Vis, @@ -2377,6 +2424,18 @@ static void PrintThreadLocalModel(GlobalVariable::ThreadLocalMode TLM, } } +static StringRef getUnnamedAddrEncoding(GlobalVariable::UnnamedAddr UA) { + switch (UA) { + case GlobalVariable::UnnamedAddr::None: + return ""; + case GlobalVariable::UnnamedAddr::Local: + return "local_unnamed_addr"; + case GlobalVariable::UnnamedAddr::Global: + return "unnamed_addr"; + } + llvm_unreachable("Unknown UnnamedAddr"); +} + static void maybePrintComdat(formatted_raw_ostream &Out, const GlobalObject &GO) { const Comdat *C = GO.getComdat(); @@ -2405,18 +2464,19 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) { if (!GV->hasInitializer() && GV->hasExternalLinkage()) Out << "external "; - PrintLinkage(GV->getLinkage(), Out); + Out << getLinkagePrintName(GV->getLinkage()); PrintVisibility(GV->getVisibility(), Out); PrintDLLStorageClass(GV->getDLLStorageClass(), Out); PrintThreadLocalModel(GV->getThreadLocalMode(), Out); - if (GV->hasUnnamedAddr()) - Out << "unnamed_addr "; + StringRef UA = getUnnamedAddrEncoding(GV->getUnnamedAddr()); + if (!UA.empty()) + Out << UA << ' '; if (unsigned AddressSpace = GV->getType()->getAddressSpace()) Out << "addrspace(" << AddressSpace << ") "; if (GV->isExternallyInitialized()) Out << "externally_initialized "; Out << (GV->isConstant() ? "constant " : "global "); - TypePrinter.print(GV->getType()->getElementType(), Out); + TypePrinter.print(GV->getValueType(), Out); if (GV->hasInitializer()) { Out << ' '; @@ -2432,39 +2492,49 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) { if (GV->getAlignment()) Out << ", align " << GV->getAlignment(); + SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; + GV->getAllMetadata(MDs); + printMetadataAttachments(MDs, ", "); + printInfoComment(*GV); } -void AssemblyWriter::printAlias(const GlobalAlias *GA) { - if (GA->isMaterializable()) +void AssemblyWriter::printIndirectSymbol(const GlobalIndirectSymbol *GIS) { + if (GIS->isMaterializable()) Out << "; Materializable\n"; - WriteAsOperandInternal(Out, GA, &TypePrinter, &Machine, GA->getParent()); + WriteAsOperandInternal(Out, GIS, &TypePrinter, &Machine, GIS->getParent()); Out << " = "; - PrintLinkage(GA->getLinkage(), Out); - PrintVisibility(GA->getVisibility(), Out); - PrintDLLStorageClass(GA->getDLLStorageClass(), Out); - PrintThreadLocalModel(GA->getThreadLocalMode(), Out); - if (GA->hasUnnamedAddr()) - Out << "unnamed_addr "; - - Out << "alias "; + Out << getLinkagePrintName(GIS->getLinkage()); + PrintVisibility(GIS->getVisibility(), Out); + PrintDLLStorageClass(GIS->getDLLStorageClass(), Out); + PrintThreadLocalModel(GIS->getThreadLocalMode(), Out); + StringRef UA = getUnnamedAddrEncoding(GIS->getUnnamedAddr()); + if (!UA.empty()) + Out << UA << ' '; + + if (isa<GlobalAlias>(GIS)) + Out << "alias "; + else if (isa<GlobalIFunc>(GIS)) + Out << "ifunc "; + else + llvm_unreachable("Not an alias or ifunc!"); - TypePrinter.print(GA->getValueType(), Out); + TypePrinter.print(GIS->getValueType(), Out); Out << ", "; - const Constant *Aliasee = GA->getAliasee(); + const Constant *IS = GIS->getIndirectSymbol(); - if (!Aliasee) { - TypePrinter.print(GA->getType(), Out); + if (!IS) { + TypePrinter.print(GIS->getType(), Out); Out << " <<NULL ALIASEE>>"; } else { - writeOperand(Aliasee, !isa<ConstantExpr>(Aliasee)); + writeOperand(IS, !isa<ConstantExpr>(IS)); } - printInfoComment(*GA); + printInfoComment(*GIS); Out << '\n'; } @@ -2544,12 +2614,18 @@ void AssemblyWriter::printFunction(const Function *F) { Out << "; Function Attrs: " << AttrStr << '\n'; } - if (F->isDeclaration()) - Out << "declare "; - else + Machine.incorporateFunction(F); + + if (F->isDeclaration()) { + Out << "declare"; + SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; + F->getAllMetadata(MDs); + printMetadataAttachments(MDs, " "); + Out << ' '; + } else Out << "define "; - PrintLinkage(F->getLinkage(), Out); + Out << getLinkagePrintName(F->getLinkage()); PrintVisibility(F->getVisibility(), Out); PrintDLLStorageClass(F->getDLLStorageClass(), Out); @@ -2566,7 +2642,6 @@ void AssemblyWriter::printFunction(const Function *F) { Out << ' '; WriteAsOperandInternal(Out, F, &TypePrinter, &Machine, F->getParent()); Out << '('; - Machine.incorporateFunction(F); // Loop over the arguments, printing them... if (F->isDeclaration() && !IsForDebug) { @@ -2598,8 +2673,9 @@ void AssemblyWriter::printFunction(const Function *F) { Out << "..."; // Output varargs portion of signature! } Out << ')'; - if (F->hasUnnamedAddr()) - Out << " unnamed_addr"; + StringRef UA = getUnnamedAddrEncoding(F->getUnnamedAddr()); + if (!UA.empty()) + Out << ' ' << UA; if (Attrs.hasAttributes(AttributeSet::FunctionIndex)) Out << " #" << Machine.getAttributeGroupSlot(Attrs.getFnAttributes()); if (F->hasSection()) { @@ -2625,17 +2701,17 @@ void AssemblyWriter::printFunction(const Function *F) { writeOperand(F->getPersonalityFn(), /*PrintType=*/true); } - SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; - F->getAllMetadata(MDs); - printMetadataAttachments(MDs, " "); - if (F->isDeclaration()) { Out << '\n'; } else { + SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; + F->getAllMetadata(MDs); + printMetadataAttachments(MDs, " "); + Out << " {"; // Output all of the function's basic blocks. - for (Function::const_iterator I = F->begin(), E = F->end(); I != E; ++I) - printBasicBlock(&*I); + for (const BasicBlock &BB : *F) + printBasicBlock(&BB); // Output the function's use-lists. printUseLists(F); @@ -2676,7 +2752,7 @@ void AssemblyWriter::printBasicBlock(const BasicBlock *BB) { Out << "\n; <label>:"; int Slot = Machine.getLocalSlot(BB); if (Slot != -1) - Out << Slot; + Out << Slot << ":"; else Out << "<badref>"; } @@ -2707,8 +2783,8 @@ void AssemblyWriter::printBasicBlock(const BasicBlock *BB) { if (AnnotationWriter) AnnotationWriter->emitBasicBlockStartAnnot(BB, Out); // Output all of the instructions in the basic block... - for (BasicBlock::const_iterator I = BB->begin(), E = BB->end(); I != E; ++I) { - printInstructionLine(*I); + for (const Instruction &I : *BB) { + printInstructionLine(I); } if (AnnotationWriter) AnnotationWriter->emitBasicBlockEndAnnot(BB, Out); @@ -3012,6 +3088,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) { Out << ' '; if (AI->isUsedWithInAlloca()) Out << "inalloca "; + if (AI->isSwiftError()) + Out << "swifterror "; TypePrinter.print(AI->getAllocatedType(), Out); // Explicitly write the array size if the code is broken, if it's an array @@ -3166,10 +3244,9 @@ void AssemblyWriter::writeAllAttributeGroups() { I != E; ++I) asVec[I->second] = *I; - for (std::vector<std::pair<AttributeSet, unsigned> >::iterator - I = asVec.begin(), E = asVec.end(); I != E; ++I) - Out << "attributes #" << I->second << " = { " - << I->first.getAsString(AttributeSet::FunctionIndex, true) << " }\n"; + for (const auto &I : asVec) + Out << "attributes #" << I.second << " = { " + << I.first.getAsString(AttributeSet::FunctionIndex, true) << " }\n"; } void AssemblyWriter::printUseListOrder(const UseListOrder &Order) { @@ -3215,6 +3292,17 @@ void AssemblyWriter::printUseLists(const Function *F) { // External Interface declarations //===----------------------------------------------------------------------===// +void Function::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, + bool ShouldPreserveUseListOrder, + bool IsForDebug) const { + SlotTracker SlotTable(this->getParent()); + formatted_raw_ostream OS(ROS); + AssemblyWriter W(OS, SlotTable, this->getParent(), AAW, + IsForDebug, + ShouldPreserveUseListOrder); + W.printFunction(this); +} + void Module::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW, bool ShouldPreserveUseListOrder, bool IsForDebug) const { SlotTracker SlotTable(this); @@ -3231,6 +3319,22 @@ void NamedMDNode::print(raw_ostream &ROS, bool IsForDebug) const { W.printNamedMDNode(this); } +void NamedMDNode::print(raw_ostream &ROS, ModuleSlotTracker &MST, + bool IsForDebug) const { + Optional<SlotTracker> LocalST; + SlotTracker *SlotTable; + if (auto *ST = MST.getMachine()) + SlotTable = ST; + else { + LocalST.emplace(getParent()); + SlotTable = &*LocalST; + } + + formatted_raw_ostream OS(ROS); + AssemblyWriter W(OS, *SlotTable, getParent(), nullptr, IsForDebug); + W.printNamedMDNode(this); +} + void Comdat::print(raw_ostream &ROS, bool /*IsForDebug*/) const { PrintLLVMName(ROS, getName(), ComdatPrefix); ROS << " = comdat "; @@ -3256,10 +3360,13 @@ void Comdat::print(raw_ostream &ROS, bool /*IsForDebug*/) const { ROS << '\n'; } -void Type::print(raw_ostream &OS, bool /*IsForDebug*/) const { +void Type::print(raw_ostream &OS, bool /*IsForDebug*/, bool NoDetails) const { TypePrinting TP; TP.print(const_cast<Type*>(this), OS); + if (NoDetails) + return; + // If the type is a named struct type, print the body as well. if (StructType *STy = dyn_cast<StructType>(const_cast<Type*>(this))) if (!STy->isLiteral()) { @@ -3316,7 +3423,7 @@ void Value::print(raw_ostream &ROS, ModuleSlotTracker &MST, else if (const Function *F = dyn_cast<Function>(GV)) W.printFunction(F); else - W.printAlias(cast<GlobalAlias>(GV)); + W.printIndirectSymbol(cast<GlobalIndirectSymbol>(GV)); } else if (const MetadataAsValue *V = dyn_cast<MetadataAsValue>(this)) { V->getMetadata()->print(ROS, MST, getModuleFromVal(V)); } else if (const Constant *C = dyn_cast<Constant>(this)) { diff --git a/lib/IR/AttributeImpl.h b/lib/IR/AttributeImpl.h index 659f9568b7c6c..267a0dab2f25b 100644 --- a/lib/IR/AttributeImpl.h +++ b/lib/IR/AttributeImpl.h @@ -17,8 +17,11 @@ #define LLVM_LIB_IR_ATTRIBUTEIMPL_H #include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/Optional.h" #include "llvm/IR/Attributes.h" +#include "llvm/Support/DataTypes.h" #include "llvm/Support/TrailingObjects.h" +#include <climits> #include <string> namespace llvm { @@ -118,7 +121,8 @@ public: : EnumAttributeImpl(IntAttrEntry, Kind), Val(Val) { assert((Kind == Attribute::Alignment || Kind == Attribute::StackAlignment || Kind == Attribute::Dereferenceable || - Kind == Attribute::DereferenceableOrNull) && + Kind == Attribute::DereferenceableOrNull || + Kind == Attribute::AllocSize) && "Wrong kind for int attribute!"); } @@ -148,19 +152,37 @@ class AttributeSetNode final friend TrailingObjects; unsigned NumAttrs; ///< Number of attributes in this node. + /// Bitset with a bit for each available attribute Attribute::AttrKind. + uint64_t AvailableAttrs; - AttributeSetNode(ArrayRef<Attribute> Attrs) : NumAttrs(Attrs.size()) { + AttributeSetNode(ArrayRef<Attribute> Attrs) + : NumAttrs(Attrs.size()), AvailableAttrs(0) { + static_assert(Attribute::EndAttrKinds <= sizeof(AvailableAttrs) * CHAR_BIT, + "Too many attributes for AvailableAttrs"); // There's memory after the node where we can store the entries in. std::copy(Attrs.begin(), Attrs.end(), getTrailingObjects<Attribute>()); + + for (Attribute I : *this) { + if (!I.isStringAttribute()) { + AvailableAttrs |= ((uint64_t)1) << I.getKindAsEnum(); + } + } } // AttributesSetNode is uniqued, these should not be publicly available. void operator=(const AttributeSetNode &) = delete; AttributeSetNode(const AttributeSetNode &) = delete; public: + void operator delete(void *p) { ::operator delete(p); } + static AttributeSetNode *get(LLVMContext &C, ArrayRef<Attribute> Attrs); - bool hasAttribute(Attribute::AttrKind Kind) const; + /// \brief Return the number of attributes this AttributeSet contains. + unsigned getNumAttributes() const { return NumAttrs; } + + bool hasAttribute(Attribute::AttrKind Kind) const { + return AvailableAttrs & ((uint64_t)1) << Kind; + } bool hasAttribute(StringRef Kind) const; bool hasAttributes() const { return NumAttrs != 0; } @@ -171,6 +193,7 @@ public: unsigned getStackAlignment() const; uint64_t getDereferenceableBytes() const; uint64_t getDereferenceableOrNullBytes() const; + std::pair<unsigned, Optional<unsigned>> getAllocSizeArgs() const; std::string getAsString(bool InAttrGrp) const; typedef const Attribute *iterator; @@ -200,10 +223,12 @@ class AttributeSetImpl final private: LLVMContext &Context; - unsigned NumAttrs; ///< Number of entries in this set. + unsigned NumSlots; ///< Number of entries in this set. + /// Bitset with a bit for each available attribute Attribute::AttrKind. + uint64_t AvailableFunctionAttrs; // Helper fn for TrailingObjects class. - size_t numTrailingObjects(OverloadToken<IndexAttrPair>) { return NumAttrs; } + size_t numTrailingObjects(OverloadToken<IndexAttrPair>) { return NumSlots; } /// \brief Return a pointer to the IndexAttrPair for the specified slot. const IndexAttrPair *getNode(unsigned Slot) const { @@ -215,27 +240,48 @@ private: AttributeSetImpl(const AttributeSetImpl &) = delete; public: AttributeSetImpl(LLVMContext &C, - ArrayRef<std::pair<unsigned, AttributeSetNode *> > Attrs) - : Context(C), NumAttrs(Attrs.size()) { + ArrayRef<std::pair<unsigned, AttributeSetNode *> > Slots) + : Context(C), NumSlots(Slots.size()), AvailableFunctionAttrs(0) { + static_assert(Attribute::EndAttrKinds <= + sizeof(AvailableFunctionAttrs) * CHAR_BIT, + "Too many attributes"); #ifndef NDEBUG - if (Attrs.size() >= 2) { - for (const std::pair<unsigned, AttributeSetNode *> *i = Attrs.begin() + 1, - *e = Attrs.end(); + if (Slots.size() >= 2) { + for (const std::pair<unsigned, AttributeSetNode *> *i = Slots.begin() + 1, + *e = Slots.end(); i != e; ++i) { assert((i-1)->first <= i->first && "Attribute set not ordered!"); } } #endif // There's memory after the node where we can store the entries in. - std::copy(Attrs.begin(), Attrs.end(), getTrailingObjects<IndexAttrPair>()); + std::copy(Slots.begin(), Slots.end(), getTrailingObjects<IndexAttrPair>()); + + // Initialize AvailableFunctionAttrs summary bitset. + if (NumSlots > 0) { + static_assert(AttributeSet::FunctionIndex == ~0u, + "FunctionIndex should be biggest possible index"); + const std::pair<unsigned, AttributeSetNode *> &Last = Slots.back(); + if (Last.first == AttributeSet::FunctionIndex) { + const AttributeSetNode *Node = Last.second; + for (Attribute I : *Node) { + if (!I.isStringAttribute()) + AvailableFunctionAttrs |= ((uint64_t)1) << I.getKindAsEnum(); + } + } + } } + void operator delete(void *p) { ::operator delete(p); } + /// \brief Get the context that created this AttributeSetImpl. LLVMContext &getContext() { return Context; } - /// \brief Return the number of attributes this AttributeSet contains. - unsigned getNumAttributes() const { return NumAttrs; } + /// \brief Return the number of slots used in this attribute list. This is + /// the number of arguments that have an attribute set on them (including the + /// function itself). + unsigned getNumSlots() const { return NumSlots; } /// \brief Get the index of the given "slot" in the AttrNodes list. This index /// is the index of the return, parameter, or function object that the @@ -258,12 +304,18 @@ public: return getNode(Slot)->second; } + /// \brief Return true if the AttributeSetNode for the FunctionIndex has an + /// enum attribute of the given kind. + bool hasFnAttribute(Attribute::AttrKind Kind) const { + return AvailableFunctionAttrs & ((uint64_t)1) << Kind; + } + typedef AttributeSetNode::iterator iterator; iterator begin(unsigned Slot) const { return getSlotNode(Slot)->begin(); } iterator end(unsigned Slot) const { return getSlotNode(Slot)->end(); } void Profile(FoldingSetNodeID &ID) const { - Profile(ID, makeArrayRef(getNode(0), getNumAttributes())); + Profile(ID, makeArrayRef(getNode(0), getNumSlots())); } static void Profile(FoldingSetNodeID &ID, ArrayRef<std::pair<unsigned, AttributeSetNode*> > Nodes) { diff --git a/lib/IR/Attributes.cpp b/lib/IR/Attributes.cpp index 6c01bb6456293..d774c1ae9dfb3 100644 --- a/lib/IR/Attributes.cpp +++ b/lib/IR/Attributes.cpp @@ -32,6 +32,35 @@ using namespace llvm; // Attribute Construction Methods //===----------------------------------------------------------------------===// +// allocsize has two integer arguments, but because they're both 32 bits, we can +// pack them into one 64-bit value, at the cost of making said value +// nonsensical. +// +// In order to do this, we need to reserve one value of the second (optional) +// allocsize argument to signify "not present." +LLVM_CONSTEXPR static unsigned AllocSizeNumElemsNotPresent = -1; + +static uint64_t packAllocSizeArgs(unsigned ElemSizeArg, + const Optional<unsigned> &NumElemsArg) { + assert((!NumElemsArg.hasValue() || + *NumElemsArg != AllocSizeNumElemsNotPresent) && + "Attempting to pack a reserved value"); + + return uint64_t(ElemSizeArg) << 32 | + NumElemsArg.getValueOr(AllocSizeNumElemsNotPresent); +} + +static std::pair<unsigned, Optional<unsigned>> +unpackAllocSizeArgs(uint64_t Num) { + unsigned NumElems = Num & std::numeric_limits<unsigned>::max(); + unsigned ElemSizeArg = Num >> 32; + + Optional<unsigned> NumElemsArg; + if (NumElems != AllocSizeNumElemsNotPresent) + NumElemsArg = NumElems; + return std::make_pair(ElemSizeArg, NumElemsArg); +} + Attribute Attribute::get(LLVMContext &Context, Attribute::AttrKind Kind, uint64_t Val) { LLVMContextImpl *pImpl = Context.pImpl; @@ -101,6 +130,14 @@ Attribute Attribute::getWithDereferenceableOrNullBytes(LLVMContext &Context, return get(Context, DereferenceableOrNull, Bytes); } +Attribute +Attribute::getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg, + const Optional<unsigned> &NumElemsArg) { + assert(!(ElemSizeArg == 0 && NumElemsArg && *NumElemsArg == 0) && + "Invalid allocsize arguments -- given allocsize(0, 0)"); + return get(Context, AllocSize, packAllocSizeArgs(ElemSizeArg, NumElemsArg)); +} + //===----------------------------------------------------------------------===// // Attribute Accessor Methods //===----------------------------------------------------------------------===// @@ -154,22 +191,18 @@ bool Attribute::hasAttribute(StringRef Kind) const { return pImpl && pImpl->hasAttribute(Kind); } -/// This returns the alignment field of an attribute as a byte alignment value. unsigned Attribute::getAlignment() const { assert(hasAttribute(Attribute::Alignment) && "Trying to get alignment from non-alignment attribute!"); return pImpl->getValueAsInt(); } -/// This returns the stack alignment field of an attribute as a byte alignment -/// value. unsigned Attribute::getStackAlignment() const { assert(hasAttribute(Attribute::StackAlignment) && "Trying to get alignment from non-alignment attribute!"); return pImpl->getValueAsInt(); } -/// This returns the number of dereferenceable bytes. uint64_t Attribute::getDereferenceableBytes() const { assert(hasAttribute(Attribute::Dereferenceable) && "Trying to get dereferenceable bytes from " @@ -184,6 +217,12 @@ uint64_t Attribute::getDereferenceableOrNullBytes() const { return pImpl->getValueAsInt(); } +std::pair<unsigned, Optional<unsigned>> Attribute::getAllocSizeArgs() const { + assert(hasAttribute(Attribute::AllocSize) && + "Trying to get allocsize args from non-allocsize attribute"); + return unpackAllocSizeArgs(pImpl->getValueAsInt()); +} + std::string Attribute::getAsString(bool InAttrGrp) const { if (!pImpl) return ""; @@ -199,6 +238,10 @@ std::string Attribute::getAsString(bool InAttrGrp) const { return "byval"; if (hasAttribute(Attribute::Convergent)) return "convergent"; + if (hasAttribute(Attribute::SwiftError)) + return "swifterror"; + if (hasAttribute(Attribute::SwiftSelf)) + return "swiftself"; if (hasAttribute(Attribute::InaccessibleMemOnly)) return "inaccessiblememonly"; if (hasAttribute(Attribute::InaccessibleMemOrArgMemOnly)) @@ -249,6 +292,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const { return "readnone"; if (hasAttribute(Attribute::ReadOnly)) return "readonly"; + if (hasAttribute(Attribute::WriteOnly)) + return "writeonly"; if (hasAttribute(Attribute::Returned)) return "returned"; if (hasAttribute(Attribute::ReturnsTwice)) @@ -312,6 +357,21 @@ std::string Attribute::getAsString(bool InAttrGrp) const { if (hasAttribute(Attribute::DereferenceableOrNull)) return AttrWithBytesToString("dereferenceable_or_null"); + if (hasAttribute(Attribute::AllocSize)) { + unsigned ElemSize; + Optional<unsigned> NumElems; + std::tie(ElemSize, NumElems) = getAllocSizeArgs(); + + std::string Result = "allocsize("; + Result += utostr(ElemSize); + if (NumElems.hasValue()) { + Result += ','; + Result += utostr(*NumElems); + } + Result += ')'; + return Result; + } + // Convert target-dependent attributes to strings of the form: // // "kind" @@ -389,7 +449,11 @@ bool AttributeImpl::operator<(const AttributeImpl &AI) const { if (isIntAttribute()) { if (AI.isEnumAttribute()) return false; - if (AI.isIntAttribute()) return getValueAsInt() < AI.getValueAsInt(); + if (AI.isIntAttribute()) { + if (getKindAsEnum() == AI.getKindAsEnum()) + return getValueAsInt() < AI.getValueAsInt(); + return getKindAsEnum() < AI.getKindAsEnum(); + } if (AI.isStringAttribute()) return true; } @@ -452,6 +516,9 @@ uint64_t AttributeImpl::getAttrMask(Attribute::AttrKind Val) { case Attribute::NoRecurse: return 1ULL << 48; case Attribute::InaccessibleMemOnly: return 1ULL << 49; case Attribute::InaccessibleMemOrArgMemOnly: return 1ULL << 50; + case Attribute::SwiftSelf: return 1ULL << 51; + case Attribute::SwiftError: return 1ULL << 52; + case Attribute::WriteOnly: return 1ULL << 53; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; @@ -462,6 +529,9 @@ uint64_t AttributeImpl::getAttrMask(Attribute::AttrKind Val) { case Attribute::ArgMemOnly: llvm_unreachable("argmemonly attribute not supported in raw format"); break; + case Attribute::AllocSize: + llvm_unreachable("allocsize not supported in raw format"); + break; } llvm_unreachable("Unsupported attribute type"); } @@ -480,7 +550,7 @@ AttributeSetNode *AttributeSetNode::get(LLVMContext &C, FoldingSetNodeID ID; SmallVector<Attribute, 8> SortedAttrs(Attrs.begin(), Attrs.end()); - array_pod_sort(SortedAttrs.begin(), SortedAttrs.end()); + std::sort(SortedAttrs.begin(), SortedAttrs.end()); for (Attribute Attr : SortedAttrs) Attr.Profile(ID); @@ -502,62 +572,65 @@ AttributeSetNode *AttributeSetNode::get(LLVMContext &C, return PA; } -bool AttributeSetNode::hasAttribute(Attribute::AttrKind Kind) const { - for (iterator I = begin(), E = end(); I != E; ++I) - if (I->hasAttribute(Kind)) - return true; - return false; -} - bool AttributeSetNode::hasAttribute(StringRef Kind) const { - for (iterator I = begin(), E = end(); I != E; ++I) - if (I->hasAttribute(Kind)) + for (Attribute I : *this) + if (I.hasAttribute(Kind)) return true; return false; } Attribute AttributeSetNode::getAttribute(Attribute::AttrKind Kind) const { - for (iterator I = begin(), E = end(); I != E; ++I) - if (I->hasAttribute(Kind)) - return *I; + if (hasAttribute(Kind)) { + for (Attribute I : *this) + if (I.hasAttribute(Kind)) + return I; + } return Attribute(); } Attribute AttributeSetNode::getAttribute(StringRef Kind) const { - for (iterator I = begin(), E = end(); I != E; ++I) - if (I->hasAttribute(Kind)) - return *I; + for (Attribute I : *this) + if (I.hasAttribute(Kind)) + return I; return Attribute(); } unsigned AttributeSetNode::getAlignment() const { - for (iterator I = begin(), E = end(); I != E; ++I) - if (I->hasAttribute(Attribute::Alignment)) - return I->getAlignment(); + for (Attribute I : *this) + if (I.hasAttribute(Attribute::Alignment)) + return I.getAlignment(); return 0; } unsigned AttributeSetNode::getStackAlignment() const { - for (iterator I = begin(), E = end(); I != E; ++I) - if (I->hasAttribute(Attribute::StackAlignment)) - return I->getStackAlignment(); + for (Attribute I : *this) + if (I.hasAttribute(Attribute::StackAlignment)) + return I.getStackAlignment(); return 0; } uint64_t AttributeSetNode::getDereferenceableBytes() const { - for (iterator I = begin(), E = end(); I != E; ++I) - if (I->hasAttribute(Attribute::Dereferenceable)) - return I->getDereferenceableBytes(); + for (Attribute I : *this) + if (I.hasAttribute(Attribute::Dereferenceable)) + return I.getDereferenceableBytes(); return 0; } uint64_t AttributeSetNode::getDereferenceableOrNullBytes() const { - for (iterator I = begin(), E = end(); I != E; ++I) - if (I->hasAttribute(Attribute::DereferenceableOrNull)) - return I->getDereferenceableOrNullBytes(); + for (Attribute I : *this) + if (I.hasAttribute(Attribute::DereferenceableOrNull)) + return I.getDereferenceableOrNullBytes(); return 0; } +std::pair<unsigned, Optional<unsigned>> +AttributeSetNode::getAllocSizeArgs() const { + for (Attribute I : *this) + if (I.hasAttribute(Attribute::AllocSize)) + return I.getAllocSizeArgs(); + return std::make_pair(0, 0); +} + std::string AttributeSetNode::getAsString(bool InAttrGrp) const { std::string Str; for (iterator I = begin(), E = end(); I != E; ++I) { @@ -573,7 +646,7 @@ std::string AttributeSetNode::getAsString(bool InAttrGrp) const { //===----------------------------------------------------------------------===// uint64_t AttributeSetImpl::Raw(unsigned Index) const { - for (unsigned I = 0, E = getNumAttributes(); I != E; ++I) { + for (unsigned I = 0, E = getNumSlots(); I != E; ++I) { if (getSlotIndex(I) != Index) continue; const AttributeSetNode *ASN = getSlotNode(I); uint64_t Mask = 0; @@ -593,6 +666,8 @@ uint64_t AttributeSetImpl::Raw(unsigned Index) const { Mask |= (Log2_32(ASN->getStackAlignment()) + 1) << 26; else if (Kind == Attribute::Dereferenceable) llvm_unreachable("dereferenceable not supported in bit mask"); + else if (Kind == Attribute::AllocSize) + llvm_unreachable("allocsize not supported in bit mask"); else Mask |= AttributeImpl::getAttrMask(Kind); } @@ -603,7 +678,7 @@ uint64_t AttributeSetImpl::Raw(unsigned Index) const { return 0; } -void AttributeSetImpl::dump() const { +LLVM_DUMP_METHOD void AttributeSetImpl::dump() const { AttributeSet(const_cast<AttributeSetImpl *>(this)).dump(); } @@ -708,6 +783,11 @@ AttributeSet AttributeSet::get(LLVMContext &C, unsigned Index, Attr = Attribute::getWithDereferenceableOrNullBytes( C, B.getDereferenceableOrNullBytes()); break; + case Attribute::AllocSize: { + auto A = B.getAllocSizeArgs(); + Attr = Attribute::getWithAllocSizeArgs(C, A.first, A.second); + break; + } default: Attr = Attribute::get(C, Kind); } @@ -715,7 +795,7 @@ AttributeSet AttributeSet::get(LLVMContext &C, unsigned Index, } // Add target-dependent (string) attributes. - for (const AttrBuilder::td_type &TDA : B.td_attrs()) + for (const auto &TDA : B.td_attrs()) Attrs.push_back( std::make_pair(Index, Attribute::get(C, TDA.first, TDA.second))); @@ -723,9 +803,17 @@ AttributeSet AttributeSet::get(LLVMContext &C, unsigned Index, } AttributeSet AttributeSet::get(LLVMContext &C, unsigned Index, - ArrayRef<Attribute::AttrKind> Kind) { + ArrayRef<Attribute::AttrKind> Kinds) { SmallVector<std::pair<unsigned, Attribute>, 8> Attrs; - for (Attribute::AttrKind K : Kind) + for (Attribute::AttrKind K : Kinds) + Attrs.push_back(std::make_pair(Index, Attribute::get(C, K))); + return get(C, Attrs); +} + +AttributeSet AttributeSet::get(LLVMContext &C, unsigned Index, + ArrayRef<StringRef> Kinds) { + SmallVector<std::pair<unsigned, Attribute>, 8> Attrs; + for (StringRef K : Kinds) Attrs.push_back(std::make_pair(Index, Attribute::get(C, K))); return get(C, Attrs); } @@ -737,7 +825,7 @@ AttributeSet AttributeSet::get(LLVMContext &C, ArrayRef<AttributeSet> Attrs) { SmallVector<std::pair<unsigned, AttributeSetNode*>, 8> AttrNodeVec; AttributeSetImpl *A0 = Attrs[0].pImpl; if (A0) - AttrNodeVec.append(A0->getNode(0), A0->getNode(A0->getNumAttributes())); + AttrNodeVec.append(A0->getNode(0), A0->getNode(A0->getNumSlots())); // Copy all attributes from Attrs into AttrNodeVec while keeping AttrNodeVec // ordered by index. Because we know that each list in Attrs is ordered by // index we only need to merge each successive list in rather than doing a @@ -748,7 +836,7 @@ AttributeSet AttributeSet::get(LLVMContext &C, ArrayRef<AttributeSet> Attrs) { SmallVector<std::pair<unsigned, AttributeSetNode *>, 8>::iterator ANVI = AttrNodeVec.begin(), ANVE; for (const IndexAttrPair *AI = AS->getNode(0), - *AE = AS->getNode(AS->getNumAttributes()); + *AE = AS->getNode(AS->getNumSlots()); AI != AE; ++AI) { ANVE = AttrNodeVec.end(); while (ANVI != ANVE && ANVI->first <= AI->first) @@ -761,16 +849,9 @@ AttributeSet AttributeSet::get(LLVMContext &C, ArrayRef<AttributeSet> Attrs) { } AttributeSet AttributeSet::addAttribute(LLVMContext &C, unsigned Index, - Attribute::AttrKind Attr) const { - if (hasAttribute(Index, Attr)) return *this; - return addAttributes(C, Index, AttributeSet::get(C, Index, Attr)); -} - -AttributeSet AttributeSet::addAttribute(LLVMContext &C, unsigned Index, - StringRef Kind) const { - llvm::AttrBuilder B; - B.addAttribute(Kind); - return addAttributes(C, Index, AttributeSet::get(C, Index, B)); + Attribute::AttrKind Kind) const { + if (hasAttribute(Index, Kind)) return *this; + return addAttributes(C, Index, AttributeSet::get(C, Index, Kind)); } AttributeSet AttributeSet::addAttribute(LLVMContext &C, unsigned Index, @@ -783,7 +864,7 @@ AttributeSet AttributeSet::addAttribute(LLVMContext &C, unsigned Index, AttributeSet AttributeSet::addAttribute(LLVMContext &C, ArrayRef<unsigned> Indices, Attribute A) const { - unsigned I = 0, E = pImpl ? pImpl->getNumAttributes() : 0; + unsigned I = 0, E = pImpl ? pImpl->getNumSlots() : 0; auto IdxI = Indices.begin(), IdxE = Indices.end(); SmallVector<AttributeSet, 4> AttrSet; @@ -826,7 +907,7 @@ AttributeSet AttributeSet::addAttributes(LLVMContext &C, unsigned Index, // Add the attribute slots before the one we're trying to add. SmallVector<AttributeSet, 4> AttrSet; - uint64_t NumAttrs = pImpl->getNumAttributes(); + uint64_t NumAttrs = pImpl->getNumSlots(); AttributeSet AS; uint64_t LastIndex = 0; for (unsigned I = 0, E = NumAttrs; I != E; ++I) { @@ -842,7 +923,7 @@ AttributeSet AttributeSet::addAttributes(LLVMContext &C, unsigned Index, // AttributeSet there. AttrBuilder B(AS, Index); - for (unsigned I = 0, E = Attrs.pImpl->getNumAttributes(); I != E; ++I) + for (unsigned I = 0, E = Attrs.pImpl->getNumSlots(); I != E; ++I) if (Attrs.getSlotIndex(I) == Index) { for (AttributeSetImpl::iterator II = Attrs.pImpl->begin(I), IE = Attrs.pImpl->end(I); II != IE; ++II) @@ -860,9 +941,15 @@ AttributeSet AttributeSet::addAttributes(LLVMContext &C, unsigned Index, } AttributeSet AttributeSet::removeAttribute(LLVMContext &C, unsigned Index, - Attribute::AttrKind Attr) const { - if (!hasAttribute(Index, Attr)) return *this; - return removeAttributes(C, Index, AttributeSet::get(C, Index, Attr)); + Attribute::AttrKind Kind) const { + if (!hasAttribute(Index, Kind)) return *this; + return removeAttributes(C, Index, AttributeSet::get(C, Index, Kind)); +} + +AttributeSet AttributeSet::removeAttribute(LLVMContext &C, unsigned Index, + StringRef Kind) const { + if (!hasAttribute(Index, Kind)) return *this; + return removeAttributes(C, Index, AttributeSet::get(C, Index, Kind)); } AttributeSet AttributeSet::removeAttributes(LLVMContext &C, unsigned Index, @@ -877,7 +964,7 @@ AttributeSet AttributeSet::removeAttributes(LLVMContext &C, unsigned Index, // Add the attribute slots before the one we're trying to add. SmallVector<AttributeSet, 4> AttrSet; - uint64_t NumAttrs = pImpl->getNumAttributes(); + uint64_t NumAttrs = pImpl->getNumSlots(); AttributeSet AS; uint64_t LastIndex = 0; for (unsigned I = 0, E = NumAttrs; I != E; ++I) { @@ -893,7 +980,7 @@ AttributeSet AttributeSet::removeAttributes(LLVMContext &C, unsigned Index, // AttributeSet there. AttrBuilder B(AS, Index); - for (unsigned I = 0, E = Attrs.pImpl->getNumAttributes(); I != E; ++I) + for (unsigned I = 0, E = Attrs.pImpl->getNumSlots(); I != E; ++I) if (Attrs.getSlotIndex(I) == Index) { B.removeAttributes(Attrs.pImpl->getSlotAttributes(I), Index); break; @@ -918,7 +1005,7 @@ AttributeSet AttributeSet::removeAttributes(LLVMContext &C, unsigned Index, // Add the attribute slots before the one we're trying to add. SmallVector<AttributeSet, 4> AttrSet; - uint64_t NumAttrs = pImpl->getNumAttributes(); + uint64_t NumAttrs = pImpl->getNumSlots(); AttributeSet AS; uint64_t LastIndex = 0; for (unsigned I = 0, E = NumAttrs; I != E; ++I) { @@ -959,6 +1046,15 @@ AttributeSet AttributeSet::addDereferenceableOrNullAttr(LLVMContext &C, return addAttributes(C, Index, AttributeSet::get(C, Index, B)); } +AttributeSet +AttributeSet::addAllocSizeAttr(LLVMContext &C, unsigned Index, + unsigned ElemSizeArg, + const Optional<unsigned> &NumElemsArg) { + llvm::AttrBuilder B; + B.addAllocSizeAttr(ElemSizeArg, NumElemsArg); + return addAttributes(C, Index, AttributeSet::get(C, Index, B)); +} + //===----------------------------------------------------------------------===// // AttributeSet Accessor Methods //===----------------------------------------------------------------------===// @@ -1008,16 +1104,21 @@ bool AttributeSet::hasAttributes(unsigned Index) const { return ASN && ASN->hasAttributes(); } -/// \brief Return true if the specified attribute is set for at least one -/// parameter or for the return value. -bool AttributeSet::hasAttrSomewhere(Attribute::AttrKind Attr) const { +bool AttributeSet::hasFnAttribute(Attribute::AttrKind Kind) const { + return pImpl && pImpl->hasFnAttribute(Kind); +} + +bool AttributeSet::hasAttrSomewhere(Attribute::AttrKind Attr, + unsigned *Index) const { if (!pImpl) return false; - for (unsigned I = 0, E = pImpl->getNumAttributes(); I != E; ++I) + for (unsigned I = 0, E = pImpl->getNumSlots(); I != E; ++I) for (AttributeSetImpl::iterator II = pImpl->begin(I), IE = pImpl->end(I); II != IE; ++II) - if (II->hasAttribute(Attr)) + if (II->hasAttribute(Attr)) { + if (Index) *Index = pImpl->getSlotIndex(I); return true; + } return false; } @@ -1054,18 +1155,22 @@ uint64_t AttributeSet::getDereferenceableOrNullBytes(unsigned Index) const { return ASN ? ASN->getDereferenceableOrNullBytes() : 0; } -std::string AttributeSet::getAsString(unsigned Index, - bool InAttrGrp) const { +std::pair<unsigned, Optional<unsigned>> +AttributeSet::getAllocSizeArgs(unsigned Index) const { + AttributeSetNode *ASN = getAttributes(Index); + return ASN ? ASN->getAllocSizeArgs() : std::make_pair(0, 0); +} + +std::string AttributeSet::getAsString(unsigned Index, bool InAttrGrp) const { AttributeSetNode *ASN = getAttributes(Index); return ASN ? ASN->getAsString(InAttrGrp) : std::string(""); } -/// \brief The attributes for the specified index are returned. AttributeSetNode *AttributeSet::getAttributes(unsigned Index) const { if (!pImpl) return nullptr; // Loop through to find the attribute node we want. - for (unsigned I = 0, E = pImpl->getNumAttributes(); I != E; ++I) + for (unsigned I = 0, E = pImpl->getNumSlots(); I != E; ++I) if (pImpl->getSlotIndex(I) == Index) return pImpl->getSlotNode(I); @@ -1088,21 +1193,18 @@ AttributeSet::iterator AttributeSet::end(unsigned Slot) const { // AttributeSet Introspection Methods //===----------------------------------------------------------------------===// -/// \brief Return the number of slots used in this attribute list. This is the -/// number of arguments that have an attribute set on them (including the -/// function itself). unsigned AttributeSet::getNumSlots() const { - return pImpl ? pImpl->getNumAttributes() : 0; + return pImpl ? pImpl->getNumSlots() : 0; } unsigned AttributeSet::getSlotIndex(unsigned Slot) const { - assert(pImpl && Slot < pImpl->getNumAttributes() && + assert(pImpl && Slot < pImpl->getNumSlots() && "Slot # out of range!"); return pImpl->getSlotIndex(Slot); } AttributeSet AttributeSet::getSlotAttributes(unsigned Slot) const { - assert(pImpl && Slot < pImpl->getNumAttributes() && + assert(pImpl && Slot < pImpl->getNumSlots() && "Slot # out of range!"); return pImpl->getSlotAttributes(Slot); } @@ -1112,7 +1214,7 @@ uint64_t AttributeSet::Raw(unsigned Index) const { return pImpl ? pImpl->Raw(Index) : 0; } -void AttributeSet::dump() const { +LLVM_DUMP_METHOD void AttributeSet::dump() const { dbgs() << "PAL[\n"; for (unsigned i = 0, e = getNumSlots(); i < e; ++i) { @@ -1134,11 +1236,11 @@ void AttributeSet::dump() const { AttrBuilder::AttrBuilder(AttributeSet AS, unsigned Index) : Attrs(0), Alignment(0), StackAlignment(0), DerefBytes(0), - DerefOrNullBytes(0) { + DerefOrNullBytes(0), AllocSizeArgs(0) { AttributeSetImpl *pImpl = AS.pImpl; if (!pImpl) return; - for (unsigned I = 0, E = pImpl->getNumAttributes(); I != E; ++I) { + for (unsigned I = 0, E = pImpl->getNumSlots(); I != E; ++I) { if (pImpl->getSlotIndex(I) != Index) continue; for (AttributeSetImpl::iterator II = pImpl->begin(I), @@ -1153,12 +1255,13 @@ void AttrBuilder::clear() { Attrs.reset(); TargetDepAttrs.clear(); Alignment = StackAlignment = DerefBytes = DerefOrNullBytes = 0; + AllocSizeArgs = 0; } AttrBuilder &AttrBuilder::addAttribute(Attribute::AttrKind Val) { assert((unsigned)Val < Attribute::EndAttrKinds && "Attribute out of range!"); assert(Val != Attribute::Alignment && Val != Attribute::StackAlignment && - Val != Attribute::Dereferenceable && + Val != Attribute::Dereferenceable && Val != Attribute::AllocSize && "Adding integer attribute without adding a value!"); Attrs[Val] = true; return *this; @@ -1181,6 +1284,8 @@ AttrBuilder &AttrBuilder::addAttribute(Attribute Attr) { DerefBytes = Attr.getDereferenceableBytes(); else if (Kind == Attribute::DereferenceableOrNull) DerefOrNullBytes = Attr.getDereferenceableOrNullBytes(); + else if (Kind == Attribute::AllocSize) + AllocSizeArgs = Attr.getValueAsInt(); return *this; } @@ -1201,6 +1306,8 @@ AttrBuilder &AttrBuilder::removeAttribute(Attribute::AttrKind Val) { DerefBytes = 0; else if (Val == Attribute::DereferenceableOrNull) DerefOrNullBytes = 0; + else if (Val == Attribute::AllocSize) + AllocSizeArgs = 0; return *this; } @@ -1235,6 +1342,10 @@ AttrBuilder &AttrBuilder::removeAttribute(StringRef A) { return *this; } +std::pair<unsigned, Optional<unsigned>> AttrBuilder::getAllocSizeArgs() const { + return unpackAllocSizeArgs(AllocSizeArgs); +} + AttrBuilder &AttrBuilder::addAlignmentAttr(unsigned Align) { if (Align == 0) return *this; @@ -1275,6 +1386,22 @@ AttrBuilder &AttrBuilder::addDereferenceableOrNullAttr(uint64_t Bytes) { return *this; } +AttrBuilder &AttrBuilder::addAllocSizeAttr(unsigned ElemSize, + const Optional<unsigned> &NumElems) { + return addAllocSizeAttrFromRawRepr(packAllocSizeArgs(ElemSize, NumElems)); +} + +AttrBuilder &AttrBuilder::addAllocSizeAttrFromRawRepr(uint64_t RawArgs) { + // (0, 0) is our "not present" value, so we need to check for it here. + assert(RawArgs && "Invalid allocsize arguments -- given allocsize(0, 0)"); + + Attrs[Attribute::AllocSize] = true; + // Reuse existing machinery to store this as a single 64-bit integer so we can + // save a few bytes over using a pair<unsigned, Optional<unsigned>>. + AllocSizeArgs = RawArgs; + return *this; +} + AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) { // FIXME: What if both have alignments, but they don't match?! if (!Alignment) @@ -1289,6 +1416,9 @@ AttrBuilder &AttrBuilder::merge(const AttrBuilder &B) { if (!DerefOrNullBytes) DerefOrNullBytes = B.DerefOrNullBytes; + if (!AllocSizeArgs) + AllocSizeArgs = B.AllocSizeArgs; + Attrs |= B.Attrs; for (auto I : B.td_attrs()) @@ -1311,6 +1441,9 @@ AttrBuilder &AttrBuilder::remove(const AttrBuilder &B) { if (B.DerefOrNullBytes) DerefOrNullBytes = 0; + if (B.AllocSizeArgs) + AllocSizeArgs = 0; + Attrs &= ~B.Attrs; for (auto I : B.td_attrs()) @@ -1389,7 +1522,8 @@ AttrBuilder &AttrBuilder::addRawValue(uint64_t Val) { I = Attribute::AttrKind(I + 1)) { if (I == Attribute::Dereferenceable || I == Attribute::DereferenceableOrNull || - I == Attribute::ArgMemOnly) + I == Attribute::ArgMemOnly || + I == Attribute::AllocSize) continue; if (uint64_t A = (Val & AttributeImpl::getAttrMask(I))) { Attrs[I] = true; @@ -1478,20 +1612,14 @@ static void adjustCallerSSPLevel(Function &Caller, const Function &Callee) { AttributeSet::FunctionIndex, B); - if (Callee.hasFnAttribute(Attribute::SafeStack)) { - Caller.removeAttributes(AttributeSet::FunctionIndex, OldSSPAttr); - Caller.addFnAttr(Attribute::SafeStack); - } else if (Callee.hasFnAttribute(Attribute::StackProtectReq) && - !Caller.hasFnAttribute(Attribute::SafeStack)) { + if (Callee.hasFnAttribute(Attribute::StackProtectReq)) { Caller.removeAttributes(AttributeSet::FunctionIndex, OldSSPAttr); Caller.addFnAttr(Attribute::StackProtectReq); } else if (Callee.hasFnAttribute(Attribute::StackProtectStrong) && - !Caller.hasFnAttribute(Attribute::SafeStack) && !Caller.hasFnAttribute(Attribute::StackProtectReq)) { Caller.removeAttributes(AttributeSet::FunctionIndex, OldSSPAttr); Caller.addFnAttr(Attribute::StackProtectStrong); } else if (Callee.hasFnAttribute(Attribute::StackProtect) && - !Caller.hasFnAttribute(Attribute::SafeStack) && !Caller.hasFnAttribute(Attribute::StackProtectReq) && !Caller.hasFnAttribute(Attribute::StackProtectStrong)) Caller.addFnAttr(Attribute::StackProtect); diff --git a/lib/IR/AutoUpgrade.cpp b/lib/IR/AutoUpgrade.cpp index 12c354c89b206..431e51bb45620 100644 --- a/lib/IR/AutoUpgrade.cpp +++ b/lib/IR/AutoUpgrade.cpp @@ -126,6 +126,10 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { StoreLaneInts[fArgs.size() - 5], Tys); return true; } + if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") { + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); + return true; + } break; } @@ -145,6 +149,31 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { break; } + case 'm': { + if (Name.startswith("masked.load.")) { + Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() }; + if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) { + F->setName(Name + ".old"); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::masked_load, + Tys); + return true; + } + } + if (Name.startswith("masked.store.")) { + auto Args = F->getFunctionType()->params(); + Type *Tys[] = { Args[0], Args[1] }; + if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) { + F->setName(Name + ".old"); + NewFn = Intrinsic::getDeclaration(F->getParent(), + Intrinsic::masked_store, + Tys); + return true; + } + } + break; + } + case 'o': // We only need to change the name to match the mangling including the // address space. @@ -159,101 +188,209 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { } break; + case 's': + if (Name == "stackprotectorcheck") { + NewFn = nullptr; + return true; + } + case 'x': { - if (Name.startswith("x86.sse2.pcmpeq.") || - Name.startswith("x86.sse2.pcmpgt.") || - Name.startswith("x86.avx2.pcmpeq.") || - Name.startswith("x86.avx2.pcmpgt.") || - Name.startswith("x86.avx2.vbroadcast") || - Name.startswith("x86.avx2.pbroadcast") || - Name.startswith("x86.avx.vpermil.") || - Name.startswith("x86.sse41.pmovsx") || - Name == "x86.avx.vinsertf128.pd.256" || - Name == "x86.avx.vinsertf128.ps.256" || - Name == "x86.avx.vinsertf128.si.256" || - Name == "x86.avx2.vinserti128" || - Name == "x86.avx.vextractf128.pd.256" || - Name == "x86.avx.vextractf128.ps.256" || - Name == "x86.avx.vextractf128.si.256" || - Name == "x86.avx2.vextracti128" || - Name == "x86.avx.movnt.dq.256" || - Name == "x86.avx.movnt.pd.256" || - Name == "x86.avx.movnt.ps.256" || - Name == "x86.sse42.crc32.64.8" || - Name == "x86.avx.vbroadcast.ss" || - Name == "x86.avx.vbroadcast.ss.256" || - Name == "x86.avx.vbroadcast.sd.256" || - Name == "x86.sse2.psll.dq" || - Name == "x86.sse2.psrl.dq" || - Name == "x86.avx2.psll.dq" || - Name == "x86.avx2.psrl.dq" || - Name == "x86.sse2.psll.dq.bs" || - Name == "x86.sse2.psrl.dq.bs" || - Name == "x86.avx2.psll.dq.bs" || - Name == "x86.avx2.psrl.dq.bs" || - Name == "x86.sse41.pblendw" || - Name == "x86.sse41.blendpd" || - Name == "x86.sse41.blendps" || - Name == "x86.avx.blend.pd.256" || - Name == "x86.avx.blend.ps.256" || - Name == "x86.avx2.pblendw" || - Name == "x86.avx2.pblendd.128" || - Name == "x86.avx2.pblendd.256" || - Name == "x86.avx2.vbroadcasti128" || - Name == "x86.xop.vpcmov" || - (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) { + bool IsX86 = Name.startswith("x86."); + if (IsX86) + Name = Name.substr(4); + + if (IsX86 && + (Name.startswith("sse2.pcmpeq.") || + Name.startswith("sse2.pcmpgt.") || + Name.startswith("avx2.pcmpeq.") || + Name.startswith("avx2.pcmpgt.") || + Name.startswith("avx512.mask.pcmpeq.") || + Name.startswith("avx512.mask.pcmpgt.") || + Name == "sse41.pmaxsb" || + Name == "sse2.pmaxs.w" || + Name == "sse41.pmaxsd" || + Name == "sse2.pmaxu.b" || + Name == "sse41.pmaxuw" || + Name == "sse41.pmaxud" || + Name == "sse41.pminsb" || + Name == "sse2.pmins.w" || + Name == "sse41.pminsd" || + Name == "sse2.pminu.b" || + Name == "sse41.pminuw" || + Name == "sse41.pminud" || + Name.startswith("avx2.pmax") || + Name.startswith("avx2.pmin") || + Name.startswith("avx2.vbroadcast") || + Name.startswith("avx2.pbroadcast") || + Name.startswith("avx.vpermil.") || + Name.startswith("sse2.pshuf") || + Name.startswith("avx512.pbroadcast") || + Name.startswith("avx512.mask.broadcast.s") || + Name.startswith("avx512.mask.movddup") || + Name.startswith("avx512.mask.movshdup") || + Name.startswith("avx512.mask.movsldup") || + Name.startswith("avx512.mask.pshuf.d.") || + Name.startswith("avx512.mask.pshufl.w.") || + Name.startswith("avx512.mask.pshufh.w.") || + Name.startswith("avx512.mask.vpermil.p") || + Name.startswith("avx512.mask.perm.df.") || + Name.startswith("avx512.mask.perm.di.") || + Name.startswith("avx512.mask.punpckl") || + Name.startswith("avx512.mask.punpckh") || + Name.startswith("avx512.mask.unpckl.") || + Name.startswith("avx512.mask.unpckh.") || + Name.startswith("avx512.mask.pand.") || + Name.startswith("avx512.mask.pandn.") || + Name.startswith("avx512.mask.por.") || + Name.startswith("avx512.mask.pxor.") || + Name.startswith("sse41.pmovsx") || + Name.startswith("sse41.pmovzx") || + Name.startswith("avx2.pmovsx") || + Name.startswith("avx2.pmovzx") || + Name == "sse2.cvtdq2pd" || + Name == "sse2.cvtps2pd" || + Name == "avx.cvtdq2.pd.256" || + Name == "avx.cvt.ps2.pd.256" || + Name == "sse2.cvttps2dq" || + Name.startswith("avx.cvtt.") || + Name.startswith("avx.vinsertf128.") || + Name == "avx2.vinserti128" || + Name.startswith("avx.vextractf128.") || + Name == "avx2.vextracti128" || + Name.startswith("sse4a.movnt.") || + Name.startswith("avx.movnt.") || + Name.startswith("avx512.storent.") || + Name == "sse2.storel.dq" || + Name.startswith("sse.storeu.") || + Name.startswith("sse2.storeu.") || + Name.startswith("avx.storeu.") || + Name.startswith("avx512.mask.storeu.p") || + Name.startswith("avx512.mask.storeu.b.") || + Name.startswith("avx512.mask.storeu.w.") || + Name.startswith("avx512.mask.storeu.d.") || + Name.startswith("avx512.mask.storeu.q.") || + Name.startswith("avx512.mask.store.p") || + Name.startswith("avx512.mask.store.b.") || + Name.startswith("avx512.mask.store.w.") || + Name.startswith("avx512.mask.store.d.") || + Name.startswith("avx512.mask.store.q.") || + Name.startswith("avx512.mask.loadu.p") || + Name.startswith("avx512.mask.loadu.b.") || + Name.startswith("avx512.mask.loadu.w.") || + Name.startswith("avx512.mask.loadu.d.") || + Name.startswith("avx512.mask.loadu.q.") || + Name.startswith("avx512.mask.load.p") || + Name.startswith("avx512.mask.load.b.") || + Name.startswith("avx512.mask.load.w.") || + Name.startswith("avx512.mask.load.d.") || + Name.startswith("avx512.mask.load.q.") || + Name == "sse42.crc32.64.8" || + Name.startswith("avx.vbroadcast.s") || + Name.startswith("avx512.mask.palignr.") || + Name.startswith("sse2.psll.dq") || + Name.startswith("sse2.psrl.dq") || + Name.startswith("avx2.psll.dq") || + Name.startswith("avx2.psrl.dq") || + Name.startswith("avx512.psll.dq") || + Name.startswith("avx512.psrl.dq") || + Name == "sse41.pblendw" || + Name.startswith("sse41.blendp") || + Name.startswith("avx.blend.p") || + Name == "avx2.pblendw" || + Name.startswith("avx2.pblendd.") || + Name == "avx2.vbroadcasti128" || + Name == "xop.vpcmov" || + (Name.startswith("xop.vpcom") && F->arg_size() == 2))) { NewFn = nullptr; return true; } // SSE4.1 ptest functions may have an old signature. - if (Name.startswith("x86.sse41.ptest")) { - if (Name == "x86.sse41.ptestc") + if (IsX86 && Name.startswith("sse41.ptest")) { + if (Name.substr(11) == "c") return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn); - if (Name == "x86.sse41.ptestz") + if (Name.substr(11) == "z") return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn); - if (Name == "x86.sse41.ptestnzc") + if (Name.substr(11) == "nzc") return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn); } // Several blend and other instructions with masks used the wrong number of // bits. - if (Name == "x86.sse41.insertps") + if (IsX86 && Name == "sse41.insertps") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps, NewFn); - if (Name == "x86.sse41.dppd") + if (IsX86 && Name == "sse41.dppd") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd, NewFn); - if (Name == "x86.sse41.dpps") + if (IsX86 && Name == "sse41.dpps") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps, NewFn); - if (Name == "x86.sse41.mpsadbw") + if (IsX86 && Name == "sse41.mpsadbw") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw, NewFn); - if (Name == "x86.avx.dp.ps.256") + if (IsX86 && Name == "avx.dp.ps.256") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256, NewFn); - if (Name == "x86.avx2.mpsadbw") + if (IsX86 && Name == "avx2.mpsadbw") return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw, NewFn); // frcz.ss/sd may need to have an argument dropped - if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) { + if (IsX86 && Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) { F->setName(Name + ".old"); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_xop_vfrcz_ss); return true; } - if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) { + if (IsX86 && Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) { F->setName(Name + ".old"); NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_xop_vfrcz_sd); return true; } + if (IsX86 && (Name.startswith("avx512.mask.pslli.") || + Name.startswith("avx512.mask.psrai.") || + Name.startswith("avx512.mask.psrli."))) { + Intrinsic::ID ShiftID; + if (Name.slice(12, 16) == "psll") + ShiftID = Name[18] == 'd' ? Intrinsic::x86_avx512_mask_psll_di_512 + : Intrinsic::x86_avx512_mask_psll_qi_512; + else if (Name.slice(12, 16) == "psra") + ShiftID = Name[18] == 'd' ? Intrinsic::x86_avx512_mask_psra_di_512 + : Intrinsic::x86_avx512_mask_psra_qi_512; + else + ShiftID = Name[18] == 'd' ? Intrinsic::x86_avx512_mask_psrl_di_512 + : Intrinsic::x86_avx512_mask_psrl_qi_512; + F->setName("llvm.x86." + Name + ".old"); + NewFn = Intrinsic::getDeclaration(F->getParent(), ShiftID); + return true; + } // Fix the FMA4 intrinsics to remove the 4 - if (Name.startswith("x86.fma4.")) { - F->setName("llvm.x86.fma" + Name.substr(8)); + if (IsX86 && Name.startswith("fma4.")) { + F->setName("llvm.x86.fma" + Name.substr(5)); NewFn = F; return true; } + // Upgrade any XOP PERMIL2 index operand still using a float/double vector. + if (IsX86 && Name.startswith("xop.vpermil2")) { + auto Params = F->getFunctionType()->params(); + auto Idx = Params[2]; + if (Idx->getScalarType()->isFloatingPointTy()) { + F->setName("llvm.x86." + Name + ".old"); + unsigned IdxSize = Idx->getPrimitiveSizeInBits(); + unsigned EltSize = Idx->getScalarSizeInBits(); + Intrinsic::ID Permil2ID; + if (EltSize == 64 && IdxSize == 128) + Permil2ID = Intrinsic::x86_xop_vpermil2pd; + else if (EltSize == 32 && IdxSize == 128) + Permil2ID = Intrinsic::x86_xop_vpermil2ps; + else if (EltSize == 64 && IdxSize == 256) + Permil2ID = Intrinsic::x86_xop_vpermil2pd_256; + else + Permil2ID = Intrinsic::x86_xop_vpermil2ps_256; + NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID); + return true; + } + } break; } } @@ -283,83 +420,219 @@ bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) { return false; } -// Handles upgrading SSE2 and AVX2 PSLLDQ intrinsics by converting them +// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them // to byte shuffles. -static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C, - Value *Op, unsigned NumLanes, - unsigned Shift) { - // Each lane is 16 bytes. - unsigned NumElts = NumLanes * 16; +static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, + Value *Op, unsigned Shift) { + Type *ResultTy = Op->getType(); + unsigned NumElts = ResultTy->getVectorNumElements() * 8; // Bitcast from a 64-bit element type to a byte element type. - Op = Builder.CreateBitCast(Op, - VectorType::get(Type::getInt8Ty(C), NumElts), - "cast"); + Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); + Op = Builder.CreateBitCast(Op, VecTy, "cast"); + // We'll be shuffling in zeroes. - Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0)); + Value *Res = Constant::getNullValue(VecTy); // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, // we'll just return the zero vector. if (Shift < 16) { - SmallVector<Constant*, 32> Idxs; - // 256-bit version is split into two 16-byte lanes. + uint32_t Idxs[64]; + // 256/512-bit version is split into 2/4 16-byte lanes. for (unsigned l = 0; l != NumElts; l += 16) for (unsigned i = 0; i != 16; ++i) { unsigned Idx = NumElts + i - Shift; if (Idx < NumElts) Idx -= NumElts - 16; // end of lane, switch operand. - Idxs.push_back(Builder.getInt32(Idx + l)); + Idxs[l + i] = Idx + l; } - Res = Builder.CreateShuffleVector(Res, Op, ConstantVector::get(Idxs)); + Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts)); } // Bitcast back to a 64-bit element type. - return Builder.CreateBitCast(Res, - VectorType::get(Type::getInt64Ty(C), 2*NumLanes), - "cast"); + return Builder.CreateBitCast(Res, ResultTy, "cast"); } -// Handles upgrading SSE2 and AVX2 PSRLDQ intrinsics by converting them +// Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them // to byte shuffles. -static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C, - Value *Op, unsigned NumLanes, +static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, unsigned Shift) { - // Each lane is 16 bytes. - unsigned NumElts = NumLanes * 16; + Type *ResultTy = Op->getType(); + unsigned NumElts = ResultTy->getVectorNumElements() * 8; // Bitcast from a 64-bit element type to a byte element type. - Op = Builder.CreateBitCast(Op, - VectorType::get(Type::getInt8Ty(C), NumElts), - "cast"); + Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); + Op = Builder.CreateBitCast(Op, VecTy, "cast"); + // We'll be shuffling in zeroes. - Value *Res = ConstantVector::getSplat(NumElts, Builder.getInt8(0)); + Value *Res = Constant::getNullValue(VecTy); // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, // we'll just return the zero vector. if (Shift < 16) { - SmallVector<Constant*, 32> Idxs; - // 256-bit version is split into two 16-byte lanes. + uint32_t Idxs[64]; + // 256/512-bit version is split into 2/4 16-byte lanes. for (unsigned l = 0; l != NumElts; l += 16) for (unsigned i = 0; i != 16; ++i) { unsigned Idx = i + Shift; if (Idx >= 16) Idx += NumElts - 16; // end of lane, switch operand. - Idxs.push_back(Builder.getInt32(Idx + l)); + Idxs[l + i] = Idx + l; } - Res = Builder.CreateShuffleVector(Op, Res, ConstantVector::get(Idxs)); + Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts)); } // Bitcast back to a 64-bit element type. - return Builder.CreateBitCast(Res, - VectorType::get(Type::getInt64Ty(C), 2*NumLanes), - "cast"); + return Builder.CreateBitCast(Res, ResultTy, "cast"); +} + +static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask, + unsigned NumElts) { + llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(), + cast<IntegerType>(Mask->getType())->getBitWidth()); + Mask = Builder.CreateBitCast(Mask, MaskTy); + + // If we have less than 8 elements, then the starting mask was an i8 and + // we need to extract down to the right number of elements. + if (NumElts < 8) { + uint32_t Indices[4]; + for (unsigned i = 0; i != NumElts; ++i) + Indices[i] = i; + Mask = Builder.CreateShuffleVector(Mask, Mask, + makeArrayRef(Indices, NumElts), + "extract"); + } + + return Mask; +} + +static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask, + Value *Op0, Value *Op1) { + // If the mask is all ones just emit the align operation. + if (const auto *C = dyn_cast<Constant>(Mask)) + if (C->isAllOnesValue()) + return Op0; + + Mask = getX86MaskVec(Builder, Mask, Op0->getType()->getVectorNumElements()); + return Builder.CreateSelect(Mask, Op0, Op1); +} + +static Value *UpgradeX86PALIGNRIntrinsics(IRBuilder<> &Builder, + Value *Op0, Value *Op1, Value *Shift, + Value *Passthru, Value *Mask) { + unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue(); + + unsigned NumElts = Op0->getType()->getVectorNumElements(); + assert(NumElts % 16 == 0); + + // If palignr is shifting the pair of vectors more than the size of two + // lanes, emit zero. + if (ShiftVal >= 32) + return llvm::Constant::getNullValue(Op0->getType()); + + // If palignr is shifting the pair of input vectors more than one lane, + // but less than two lanes, convert to shifting in zeroes. + if (ShiftVal > 16) { + ShiftVal -= 16; + Op1 = Op0; + Op0 = llvm::Constant::getNullValue(Op0->getType()); + } + + uint32_t Indices[64]; + // 256-bit palignr operates on 128-bit lanes so we need to handle that + for (unsigned l = 0; l != NumElts; l += 16) { + for (unsigned i = 0; i != 16; ++i) { + unsigned Idx = ShiftVal + i; + if (Idx >= 16) + Idx += NumElts - 16; // End of lane, switch operand. + Indices[l + i] = Idx + l; + } + } + + Value *Align = Builder.CreateShuffleVector(Op1, Op0, + makeArrayRef(Indices, NumElts), + "palignr"); + + return EmitX86Select(Builder, Mask, Align, Passthru); +} + +static Value *UpgradeMaskedStore(IRBuilder<> &Builder, + Value *Ptr, Value *Data, Value *Mask, + bool Aligned) { + // Cast the pointer to the right type. + Ptr = Builder.CreateBitCast(Ptr, + llvm::PointerType::getUnqual(Data->getType())); + unsigned Align = + Aligned ? cast<VectorType>(Data->getType())->getBitWidth() / 8 : 1; + + // If the mask is all ones just emit a regular store. + if (const auto *C = dyn_cast<Constant>(Mask)) + if (C->isAllOnesValue()) + return Builder.CreateAlignedStore(Data, Ptr, Align); + + // Convert the mask from an integer type to a vector of i1. + unsigned NumElts = Data->getType()->getVectorNumElements(); + Mask = getX86MaskVec(Builder, Mask, NumElts); + return Builder.CreateMaskedStore(Data, Ptr, Align, Mask); +} + +static Value *UpgradeMaskedLoad(IRBuilder<> &Builder, + Value *Ptr, Value *Passthru, Value *Mask, + bool Aligned) { + // Cast the pointer to the right type. + Ptr = Builder.CreateBitCast(Ptr, + llvm::PointerType::getUnqual(Passthru->getType())); + unsigned Align = + Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1; + + // If the mask is all ones just emit a regular store. + if (const auto *C = dyn_cast<Constant>(Mask)) + if (C->isAllOnesValue()) + return Builder.CreateAlignedLoad(Ptr, Align); + + // Convert the mask from an integer type to a vector of i1. + unsigned NumElts = Passthru->getType()->getVectorNumElements(); + Mask = getX86MaskVec(Builder, Mask, NumElts); + return Builder.CreateMaskedLoad(Ptr, Align, Mask, Passthru); } -// UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the -// upgraded intrinsic. All argument and return casting must be provided in -// order to seamlessly integrate with existing context. +static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI, + ICmpInst::Predicate Pred) { + Value *Op0 = CI.getArgOperand(0); + Value *Op1 = CI.getArgOperand(1); + Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1); + return Builder.CreateSelect(Cmp, Op0, Op1); +} + +static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI, + ICmpInst::Predicate Pred) { + Value *Op0 = CI.getArgOperand(0); + unsigned NumElts = Op0->getType()->getVectorNumElements(); + Value *Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1)); + + Value *Mask = CI.getArgOperand(2); + const auto *C = dyn_cast<Constant>(Mask); + if (!C || !C->isAllOnesValue()) + Cmp = Builder.CreateAnd(Cmp, getX86MaskVec(Builder, Mask, NumElts)); + + if (NumElts < 8) { + uint32_t Indices[8]; + for (unsigned i = 0; i != NumElts; ++i) + Indices[i] = i; + for (unsigned i = NumElts; i != 8; ++i) + Indices[i] = NumElts + i % NumElts; + Cmp = Builder.CreateShuffleVector(Cmp, + Constant::getNullValue(Cmp->getType()), + Indices); + } + return Builder.CreateBitCast(Cmp, IntegerType::get(CI.getContext(), + std::max(NumElts, 8U))); +} + +/// Upgrade a call to an old intrinsic. All argument and return casting must be +/// provided to seamlessly integrate with existing context. void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Function *F = CI->getCalledFunction(); LLVMContext &C = CI->getContext(); @@ -372,26 +645,105 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Get the Function's name. StringRef Name = F->getName(); + assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'"); + Name = Name.substr(5); + + bool IsX86 = Name.startswith("x86."); + if (IsX86) + Name = Name.substr(4); + Value *Rep; - // Upgrade packed integer vector compares intrinsics to compare instructions - if (Name.startswith("llvm.x86.sse2.pcmpeq.") || - Name.startswith("llvm.x86.avx2.pcmpeq.")) { + // Upgrade packed integer vector compare intrinsics to compare instructions. + if (IsX86 && (Name.startswith("sse2.pcmpeq.") || + Name.startswith("avx2.pcmpeq."))) { Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1), "pcmpeq"); - // need to sign extend since icmp returns vector of i1 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); - } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") || - Name.startswith("llvm.x86.avx2.pcmpgt.")) { + } else if (IsX86 && (Name.startswith("sse2.pcmpgt.") || + Name.startswith("avx2.pcmpgt."))) { Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1), "pcmpgt"); - // need to sign extend since icmp returns vector of i1 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); - } else if (Name == "llvm.x86.avx.movnt.dq.256" || - Name == "llvm.x86.avx.movnt.ps.256" || - Name == "llvm.x86.avx.movnt.pd.256") { - IRBuilder<> Builder(C); - Builder.SetInsertPoint(CI->getParent(), CI->getIterator()); + } else if (IsX86 && Name.startswith("avx512.mask.pcmpeq.")) { + Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_EQ); + } else if (IsX86 && Name.startswith("avx512.mask.pcmpgt.")) { + Rep = upgradeMaskedCompare(Builder, *CI, ICmpInst::ICMP_SGT); + } else if (IsX86 && (Name == "sse41.pmaxsb" || + Name == "sse2.pmaxs.w" || + Name == "sse41.pmaxsd" || + Name.startswith("avx2.pmaxs"))) { + Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT); + } else if (IsX86 && (Name == "sse2.pmaxu.b" || + Name == "sse41.pmaxuw" || + Name == "sse41.pmaxud" || + Name.startswith("avx2.pmaxu"))) { + Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT); + } else if (IsX86 && (Name == "sse41.pminsb" || + Name == "sse2.pmins.w" || + Name == "sse41.pminsd" || + Name.startswith("avx2.pmins"))) { + Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT); + } else if (IsX86 && (Name == "sse2.pminu.b" || + Name == "sse41.pminuw" || + Name == "sse41.pminud" || + Name.startswith("avx2.pminu"))) { + Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT); + } else if (IsX86 && (Name == "sse2.cvtdq2pd" || + Name == "sse2.cvtps2pd" || + Name == "avx.cvtdq2.pd.256" || + Name == "avx.cvt.ps2.pd.256")) { + // Lossless i32/float to double conversion. + // Extract the bottom elements if necessary and convert to double vector. + Value *Src = CI->getArgOperand(0); + VectorType *SrcTy = cast<VectorType>(Src->getType()); + VectorType *DstTy = cast<VectorType>(CI->getType()); + Rep = CI->getArgOperand(0); + + unsigned NumDstElts = DstTy->getNumElements(); + if (NumDstElts < SrcTy->getNumElements()) { + assert(NumDstElts == 2 && "Unexpected vector size"); + uint32_t ShuffleMask[2] = { 0, 1 }; + Rep = Builder.CreateShuffleVector(Rep, UndefValue::get(SrcTy), + ShuffleMask); + } + + bool Int2Double = (StringRef::npos != Name.find("cvtdq2")); + if (Int2Double) + Rep = Builder.CreateSIToFP(Rep, DstTy, "cvtdq2pd"); + else + Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd"); + } else if (IsX86 && (Name == "sse2.cvttps2dq" || + Name.startswith("avx.cvtt."))) { + // Truncation (round to zero) float/double to i32 vector conversion. + Value *Src = CI->getArgOperand(0); + VectorType *DstTy = cast<VectorType>(CI->getType()); + Rep = Builder.CreateFPToSI(Src, DstTy, "cvtt"); + } else if (IsX86 && Name.startswith("sse4a.movnt.")) { + Module *M = F->getParent(); + SmallVector<Metadata *, 1> Elts; + Elts.push_back( + ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); + MDNode *Node = MDNode::get(C, Elts); + + Value *Arg0 = CI->getArgOperand(0); + Value *Arg1 = CI->getArgOperand(1); + + // Nontemporal (unaligned) store of the 0'th element of the float/double + // vector. + Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType(); + PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy); + Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast"); + Value *Extract = + Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement"); + + StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, 1); + SI->setMetadata(M->getMDKindID("nontemporal"), Node); + // Remove intrinsic. + CI->eraseFromParent(); + return; + } else if (IsX86 && (Name.startswith("avx.movnt.") || + Name.startswith("avx512.storent."))) { Module *M = F->getParent(); SmallVector<Metadata *, 1> Elts; Elts.push_back( @@ -405,14 +757,82 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *BC = Builder.CreateBitCast(Arg0, PointerType::getUnqual(Arg1->getType()), "cast"); - StoreInst *SI = Builder.CreateStore(Arg1, BC); + VectorType *VTy = cast<VectorType>(Arg1->getType()); + StoreInst *SI = Builder.CreateAlignedStore(Arg1, BC, + VTy->getBitWidth() / 8); SI->setMetadata(M->getMDKindID("nontemporal"), Node); - SI->setAlignment(32); // Remove intrinsic. CI->eraseFromParent(); return; - } else if (Name.startswith("llvm.x86.xop.vpcom")) { + } else if (IsX86 && Name == "sse2.storel.dq") { + Value *Arg0 = CI->getArgOperand(0); + Value *Arg1 = CI->getArgOperand(1); + + Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2); + Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast"); + Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0); + Value *BC = Builder.CreateBitCast(Arg0, + PointerType::getUnqual(Elt->getType()), + "cast"); + Builder.CreateAlignedStore(Elt, BC, 1); + + // Remove intrinsic. + CI->eraseFromParent(); + return; + } else if (IsX86 && (Name.startswith("sse.storeu.") || + Name.startswith("sse2.storeu.") || + Name.startswith("avx.storeu."))) { + Value *Arg0 = CI->getArgOperand(0); + Value *Arg1 = CI->getArgOperand(1); + + Arg0 = Builder.CreateBitCast(Arg0, + PointerType::getUnqual(Arg1->getType()), + "cast"); + Builder.CreateAlignedStore(Arg1, Arg0, 1); + + // Remove intrinsic. + CI->eraseFromParent(); + return; + } else if (IsX86 && (Name.startswith("avx512.mask.storeu.p") || + Name.startswith("avx512.mask.storeu.b.") || + Name.startswith("avx512.mask.storeu.w.") || + Name.startswith("avx512.mask.storeu.d.") || + Name.startswith("avx512.mask.storeu.q."))) { + UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), + CI->getArgOperand(2), /*Aligned*/false); + + // Remove intrinsic. + CI->eraseFromParent(); + return; + } else if (IsX86 && (Name.startswith("avx512.mask.store.p") || + Name.startswith("avx512.mask.store.b.") || + Name.startswith("avx512.mask.store.w.") || + Name.startswith("avx512.mask.store.d.") || + Name.startswith("avx512.mask.store.q."))) { + UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), + CI->getArgOperand(2), /*Aligned*/true); + + // Remove intrinsic. + CI->eraseFromParent(); + return; + } else if (IsX86 && (Name.startswith("avx512.mask.loadu.p") || + Name.startswith("avx512.mask.loadu.b.") || + Name.startswith("avx512.mask.loadu.w.") || + Name.startswith("avx512.mask.loadu.d.") || + Name.startswith("avx512.mask.loadu.q."))) { + Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), + CI->getArgOperand(1), CI->getArgOperand(2), + /*Aligned*/false); + } else if (IsX86 && (Name.startswith("avx512.mask.load.p") || + Name.startswith("avx512.mask.load.b.") || + Name.startswith("avx512.mask.load.w.") || + Name.startswith("avx512.mask.load.d.") || + Name.startswith("avx512.mask.load.q."))) { + Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), + CI->getArgOperand(1),CI->getArgOperand(2), + /*Aligned*/true); + } else if (IsX86 && Name.startswith("xop.vpcom")) { Intrinsic::ID intID; if (Name.endswith("ub")) intID = Intrinsic::x86_xop_vpcomub; @@ -433,7 +853,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { else llvm_unreachable("Unknown suffix"); - Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom" + Name = Name.substr(9); // strip off "xop.vpcom" unsigned Imm; if (Name.startswith("lt")) Imm = 0; @@ -458,7 +878,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Rep = Builder.CreateCall(VPCOM, {CI->getArgOperand(0), CI->getArgOperand(1), Builder.getInt8(Imm)}); - } else if (Name == "llvm.x86.xop.vpcmov") { + } else if (IsX86 && Name == "xop.vpcmov") { Value *Arg0 = CI->getArgOperand(0); Value *Arg1 = CI->getArgOperand(1); Value *Sel = CI->getArgOperand(2); @@ -468,13 +888,13 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Value *Sel0 = Builder.CreateAnd(Arg0, Sel); Value *Sel1 = Builder.CreateAnd(Arg1, NotSel); Rep = Builder.CreateOr(Sel0, Sel1); - } else if (Name == "llvm.x86.sse42.crc32.64.8") { + } else if (IsX86 && Name == "sse42.crc32.64.8") { Function *CRC32 = Intrinsic::getDeclaration(F->getParent(), Intrinsic::x86_sse42_crc32_32_8); Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)}); Rep = Builder.CreateZExt(Rep, CI->getType(), ""); - } else if (Name.startswith("llvm.x86.avx.vbroadcast")) { + } else if (IsX86 && Name.startswith("avx.vbroadcast")) { // Replace broadcasts with a series of insertelements. Type *VecTy = CI->getType(); Type *EltTy = VecTy->getVectorElementType(); @@ -487,101 +907,96 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { for (unsigned I = 0; I < EltNum; ++I) Rep = Builder.CreateInsertElement(Rep, Load, ConstantInt::get(I32Ty, I)); - } else if (Name.startswith("llvm.x86.sse41.pmovsx")) { + } else if (IsX86 && (Name.startswith("sse41.pmovsx") || + Name.startswith("sse41.pmovzx") || + Name.startswith("avx2.pmovsx") || + Name.startswith("avx2.pmovzx"))) { VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType()); VectorType *DstTy = cast<VectorType>(CI->getType()); unsigned NumDstElts = DstTy->getNumElements(); - // Extract a subvector of the first NumDstElts lanes and sign extend. - SmallVector<int, 8> ShuffleMask; - for (int i = 0; i != (int)NumDstElts; ++i) - ShuffleMask.push_back(i); + // Extract a subvector of the first NumDstElts lanes and sign/zero extend. + SmallVector<uint32_t, 8> ShuffleMask(NumDstElts); + for (unsigned i = 0; i != NumDstElts; ++i) + ShuffleMask[i] = i; Value *SV = Builder.CreateShuffleVector( CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask); - Rep = Builder.CreateSExt(SV, DstTy); - } else if (Name == "llvm.x86.avx2.vbroadcasti128") { + + bool DoSext = (StringRef::npos != Name.find("pmovsx")); + Rep = DoSext ? Builder.CreateSExt(SV, DstTy) + : Builder.CreateZExt(SV, DstTy); + } else if (IsX86 && Name == "avx2.vbroadcasti128") { // Replace vbroadcasts with a vector shuffle. Type *VT = VectorType::get(Type::getInt64Ty(C), 2); Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), PointerType::getUnqual(VT)); Value *Load = Builder.CreateLoad(VT, Op); - const int Idxs[4] = { 0, 1, 0, 1 }; + uint32_t Idxs[4] = { 0, 1, 0, 1 }; Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), Idxs); - } else if (Name.startswith("llvm.x86.avx2.pbroadcast") || - Name.startswith("llvm.x86.avx2.vbroadcast")) { + } else if (IsX86 && (Name.startswith("avx2.pbroadcast") || + Name.startswith("avx2.vbroadcast") || + Name.startswith("avx512.pbroadcast") || + Name.startswith("avx512.mask.broadcast.s"))) { // Replace vp?broadcasts with a vector shuffle. Value *Op = CI->getArgOperand(0); unsigned NumElts = CI->getType()->getVectorNumElements(); Type *MaskTy = VectorType::get(Type::getInt32Ty(C), NumElts); Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()), Constant::getNullValue(MaskTy)); - } else if (Name == "llvm.x86.sse2.psll.dq") { - // 128-bit shift left specified in bits. - unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1, - Shift / 8); // Shift is in bits. - } else if (Name == "llvm.x86.sse2.psrl.dq") { - // 128-bit shift right specified in bits. - unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1, - Shift / 8); // Shift is in bits. - } else if (Name == "llvm.x86.avx2.psll.dq") { - // 256-bit shift left specified in bits. + + if (CI->getNumArgOperands() == 3) + Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, + CI->getArgOperand(1)); + } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) { + Rep = UpgradeX86PALIGNRIntrinsics(Builder, CI->getArgOperand(0), + CI->getArgOperand(1), + CI->getArgOperand(2), + CI->getArgOperand(3), + CI->getArgOperand(4)); + } else if (IsX86 && (Name == "sse2.psll.dq" || + Name == "avx2.psll.dq")) { + // 128/256-bit shift left specified in bits. unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2, + Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift / 8); // Shift is in bits. - } else if (Name == "llvm.x86.avx2.psrl.dq") { - // 256-bit shift right specified in bits. + } else if (IsX86 && (Name == "sse2.psrl.dq" || + Name == "avx2.psrl.dq")) { + // 128/256-bit shift right specified in bits. unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2, + Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift / 8); // Shift is in bits. - } else if (Name == "llvm.x86.sse2.psll.dq.bs") { - // 128-bit shift left specified in bytes. - unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1, - Shift); - } else if (Name == "llvm.x86.sse2.psrl.dq.bs") { - // 128-bit shift right specified in bytes. - unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 1, - Shift); - } else if (Name == "llvm.x86.avx2.psll.dq.bs") { - // 256-bit shift left specified in bytes. + } else if (IsX86 && (Name == "sse2.psll.dq.bs" || + Name == "avx2.psll.dq.bs" || + Name == "avx512.psll.dq.512")) { + // 128/256/512-bit shift left specified in bytes. unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - Rep = UpgradeX86PSLLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2, - Shift); - } else if (Name == "llvm.x86.avx2.psrl.dq.bs") { - // 256-bit shift right specified in bytes. + Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); + } else if (IsX86 && (Name == "sse2.psrl.dq.bs" || + Name == "avx2.psrl.dq.bs" || + Name == "avx512.psrl.dq.512")) { + // 128/256/512-bit shift right specified in bytes. unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - Rep = UpgradeX86PSRLDQIntrinsics(Builder, C, CI->getArgOperand(0), 2, - Shift); - } else if (Name == "llvm.x86.sse41.pblendw" || - Name == "llvm.x86.sse41.blendpd" || - Name == "llvm.x86.sse41.blendps" || - Name == "llvm.x86.avx.blend.pd.256" || - Name == "llvm.x86.avx.blend.ps.256" || - Name == "llvm.x86.avx2.pblendw" || - Name == "llvm.x86.avx2.pblendd.128" || - Name == "llvm.x86.avx2.pblendd.256") { + Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); + } else if (IsX86 && (Name == "sse41.pblendw" || + Name.startswith("sse41.blendp") || + Name.startswith("avx.blend.p") || + Name == "avx2.pblendw" || + Name.startswith("avx2.pblendd."))) { Value *Op0 = CI->getArgOperand(0); Value *Op1 = CI->getArgOperand(1); unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue(); VectorType *VecTy = cast<VectorType>(CI->getType()); unsigned NumElts = VecTy->getNumElements(); - SmallVector<Constant*, 16> Idxs; - for (unsigned i = 0; i != NumElts; ++i) { - unsigned Idx = ((Imm >> (i%8)) & 1) ? i + NumElts : i; - Idxs.push_back(Builder.getInt32(Idx)); - } + SmallVector<uint32_t, 16> Idxs(NumElts); + for (unsigned i = 0; i != NumElts; ++i) + Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i; - Rep = Builder.CreateShuffleVector(Op0, Op1, ConstantVector::get(Idxs)); - } else if (Name == "llvm.x86.avx.vinsertf128.pd.256" || - Name == "llvm.x86.avx.vinsertf128.ps.256" || - Name == "llvm.x86.avx.vinsertf128.si.256" || - Name == "llvm.x86.avx2.vinserti128") { + Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); + } else if (IsX86 && (Name.startswith("avx.vinsertf128.") || + Name == "avx2.vinserti128")) { Value *Op0 = CI->getArgOperand(0); Value *Op1 = CI->getArgOperand(1); unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); @@ -593,11 +1008,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Extend the second operand into a vector that is twice as big. Value *UndefV = UndefValue::get(Op1->getType()); - SmallVector<Constant*, 8> Idxs; - for (unsigned i = 0; i != NumElts; ++i) { - Idxs.push_back(Builder.getInt32(i)); - } - Rep = Builder.CreateShuffleVector(Op1, UndefV, ConstantVector::get(Idxs)); + SmallVector<uint32_t, 8> Idxs(NumElts); + for (unsigned i = 0; i != NumElts; ++i) + Idxs[i] = i; + Rep = Builder.CreateShuffleVector(Op1, UndefV, Idxs); // Insert the second operand into the first operand. @@ -610,24 +1024,17 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11> // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 > - SmallVector<Constant*, 8> Idxs2; // The low half of the result is either the low half of the 1st operand // or the low half of the 2nd operand (the inserted vector). - for (unsigned i = 0; i != NumElts / 2; ++i) { - unsigned Idx = Imm ? i : (i + NumElts); - Idxs2.push_back(Builder.getInt32(Idx)); - } + for (unsigned i = 0; i != NumElts / 2; ++i) + Idxs[i] = Imm ? i : (i + NumElts); // The high half of the result is either the low half of the 2nd operand // (the inserted vector) or the high half of the 1st operand. - for (unsigned i = NumElts / 2; i != NumElts; ++i) { - unsigned Idx = Imm ? (i + NumElts / 2) : i; - Idxs2.push_back(Builder.getInt32(Idx)); - } - Rep = Builder.CreateShuffleVector(Op0, Rep, ConstantVector::get(Idxs2)); - } else if (Name == "llvm.x86.avx.vextractf128.pd.256" || - Name == "llvm.x86.avx.vextractf128.ps.256" || - Name == "llvm.x86.avx.vextractf128.si.256" || - Name == "llvm.x86.avx2.vextracti128") { + for (unsigned i = NumElts / 2; i != NumElts; ++i) + Idxs[i] = Imm ? (i + NumElts / 2) : i; + Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs); + } else if (IsX86 && (Name.startswith("avx.vextractf128.") || + Name == "avx2.vextracti128")) { Value *Op0 = CI->getArgOperand(0); unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); VectorType *VecTy = cast<VectorType>(CI->getType()); @@ -637,54 +1044,170 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { Imm = Imm & 1; // Get indexes for either the high half or low half of the input vector. - SmallVector<Constant*, 4> Idxs(NumElts); + SmallVector<uint32_t, 4> Idxs(NumElts); for (unsigned i = 0; i != NumElts; ++i) { - unsigned Idx = Imm ? (i + NumElts) : i; - Idxs[i] = Builder.getInt32(Idx); + Idxs[i] = Imm ? (i + NumElts) : i; } Value *UndefV = UndefValue::get(Op0->getType()); - Rep = Builder.CreateShuffleVector(Op0, UndefV, ConstantVector::get(Idxs)); - } else { - bool PD128 = false, PD256 = false, PS128 = false, PS256 = false; - if (Name == "llvm.x86.avx.vpermil.pd.256") - PD256 = true; - else if (Name == "llvm.x86.avx.vpermil.pd") - PD128 = true; - else if (Name == "llvm.x86.avx.vpermil.ps.256") - PS256 = true; - else if (Name == "llvm.x86.avx.vpermil.ps") - PS128 = true; - - if (PD256 || PD128 || PS256 || PS128) { - Value *Op0 = CI->getArgOperand(0); - unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); - SmallVector<Constant*, 8> Idxs; - - if (PD128) - for (unsigned i = 0; i != 2; ++i) - Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1)); - else if (PD256) - for (unsigned l = 0; l != 4; l+=2) - for (unsigned i = 0; i != 2; ++i) - Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l)); - else if (PS128) - for (unsigned i = 0; i != 4; ++i) - Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3)); - else if (PS256) - for (unsigned l = 0; l != 8; l+=4) - for (unsigned i = 0; i != 4; ++i) - Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l)); - else - llvm_unreachable("Unexpected function"); + Rep = Builder.CreateShuffleVector(Op0, UndefV, Idxs); + } else if (!IsX86 && Name == "stackprotectorcheck") { + Rep = nullptr; + } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") || + Name.startswith("avx512.mask.perm.di."))) { + Value *Op0 = CI->getArgOperand(0); + unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); + VectorType *VecTy = cast<VectorType>(CI->getType()); + unsigned NumElts = VecTy->getNumElements(); - Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs)); - } else { - llvm_unreachable("Unknown function for CallInst upgrade."); + SmallVector<uint32_t, 8> Idxs(NumElts); + for (unsigned i = 0; i != NumElts; ++i) + Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3); + + Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); + + if (CI->getNumArgOperands() == 4) + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && (Name.startswith("avx.vpermil.") || + Name == "sse2.pshuf.d" || + Name.startswith("avx512.mask.vpermil.p") || + Name.startswith("avx512.mask.pshuf.d."))) { + Value *Op0 = CI->getArgOperand(0); + unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); + VectorType *VecTy = cast<VectorType>(CI->getType()); + unsigned NumElts = VecTy->getNumElements(); + // Calculate the size of each index in the immediate. + unsigned IdxSize = 64 / VecTy->getScalarSizeInBits(); + unsigned IdxMask = ((1 << IdxSize) - 1); + + SmallVector<uint32_t, 8> Idxs(NumElts); + // Lookup the bits for this element, wrapping around the immediate every + // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need + // to offset by the first index of each group. + for (unsigned i = 0; i != NumElts; ++i) + Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask); + + Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); + + if (CI->getNumArgOperands() == 4) + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && (Name == "sse2.pshufl.w" || + Name.startswith("avx512.mask.pshufl.w."))) { + Value *Op0 = CI->getArgOperand(0); + unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); + unsigned NumElts = CI->getType()->getVectorNumElements(); + + SmallVector<uint32_t, 16> Idxs(NumElts); + for (unsigned l = 0; l != NumElts; l += 8) { + for (unsigned i = 0; i != 4; ++i) + Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l; + for (unsigned i = 4; i != 8; ++i) + Idxs[i + l] = i + l; + } + + Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); + + if (CI->getNumArgOperands() == 4) + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && (Name == "sse2.pshufh.w" || + Name.startswith("avx512.mask.pshufh.w."))) { + Value *Op0 = CI->getArgOperand(0); + unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); + unsigned NumElts = CI->getType()->getVectorNumElements(); + + SmallVector<uint32_t, 16> Idxs(NumElts); + for (unsigned l = 0; l != NumElts; l += 8) { + for (unsigned i = 0; i != 4; ++i) + Idxs[i + l] = i + l; + for (unsigned i = 0; i != 4; ++i) + Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l; } + + Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); + + if (CI->getNumArgOperands() == 4) + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && (Name.startswith("avx512.mask.movddup") || + Name.startswith("avx512.mask.movshdup") || + Name.startswith("avx512.mask.movsldup"))) { + Value *Op0 = CI->getArgOperand(0); + unsigned NumElts = CI->getType()->getVectorNumElements(); + unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); + + unsigned Offset = 0; + if (Name.startswith("avx512.mask.movshdup.")) + Offset = 1; + + SmallVector<uint32_t, 16> Idxs(NumElts); + for (unsigned l = 0; l != NumElts; l += NumLaneElts) + for (unsigned i = 0; i != NumLaneElts; i += 2) { + Idxs[i + l + 0] = i + l + Offset; + Idxs[i + l + 1] = i + l + Offset; + } + + Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); + + Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, + CI->getArgOperand(1)); + } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") || + Name.startswith("avx512.mask.unpckl."))) { + Value *Op0 = CI->getArgOperand(0); + Value *Op1 = CI->getArgOperand(1); + int NumElts = CI->getType()->getVectorNumElements(); + int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); + + SmallVector<uint32_t, 64> Idxs(NumElts); + for (int l = 0; l != NumElts; l += NumLaneElts) + for (int i = 0; i != NumLaneElts; ++i) + Idxs[i + l] = l + (i / 2) + NumElts * (i % 2); + + Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); + + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") || + Name.startswith("avx512.mask.unpckh."))) { + Value *Op0 = CI->getArgOperand(0); + Value *Op1 = CI->getArgOperand(1); + int NumElts = CI->getType()->getVectorNumElements(); + int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); + + SmallVector<uint32_t, 64> Idxs(NumElts); + for (int l = 0; l != NumElts; l += NumLaneElts) + for (int i = 0; i != NumLaneElts; ++i) + Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2); + + Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); + + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.pand.")) { + Rep = Builder.CreateAnd(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.pandn.")) { + Rep = Builder.CreateAnd(Builder.CreateNot(CI->getArgOperand(0)), + CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.por.")) { + Rep = Builder.CreateOr(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else if (IsX86 && Name.startswith("avx512.mask.pxor.")) { + Rep = Builder.CreateXor(CI->getArgOperand(0), CI->getArgOperand(1)); + Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, + CI->getArgOperand(2)); + } else { + llvm_unreachable("Unknown function for CallInst upgrade."); } - CI->replaceAllUsesWith(Rep); + if (Rep) + CI->replaceAllUsesWith(Rep); CI->eraseFromParent(); return; } @@ -697,6 +1220,12 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { default: llvm_unreachable("Unknown function for CallInst upgrade."); + case Intrinsic::x86_avx512_mask_psll_di_512: + case Intrinsic::x86_avx512_mask_psra_di_512: + case Intrinsic::x86_avx512_mask_psrl_di_512: + case Intrinsic::x86_avx512_mask_psll_qi_512: + case Intrinsic::x86_avx512_mask_psra_qi_512: + case Intrinsic::x86_avx512_mask_psrl_qi_512: case Intrinsic::arm_neon_vld1: case Intrinsic::arm_neon_vld2: case Intrinsic::arm_neon_vld3: @@ -746,6 +1275,20 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { CI->eraseFromParent(); return; + case Intrinsic::x86_xop_vpermil2pd: + case Intrinsic::x86_xop_vpermil2ps: + case Intrinsic::x86_xop_vpermil2pd_256: + case Intrinsic::x86_xop_vpermil2ps_256: { + SmallVector<Value *, 4> Args(CI->arg_operands().begin(), + CI->arg_operands().end()); + VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType()); + VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy); + Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy); + CI->replaceAllUsesWith(Builder.CreateCall(NewFn, Args, Name)); + CI->eraseFromParent(); + return; + } + case Intrinsic::x86_sse41_ptestc: case Intrinsic::x86_sse41_ptestz: case Intrinsic::x86_sse41_ptestnzc: { @@ -790,24 +1333,37 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { CI->eraseFromParent(); return; } + + case Intrinsic::thread_pointer: { + CI->replaceAllUsesWith(Builder.CreateCall(NewFn, {})); + CI->eraseFromParent(); + return; + } + + case Intrinsic::masked_load: + case Intrinsic::masked_store: { + SmallVector<Value *, 4> Args(CI->arg_operands().begin(), + CI->arg_operands().end()); + CI->replaceAllUsesWith(Builder.CreateCall(NewFn, Args)); + CI->eraseFromParent(); + return; + } } } -// This tests each Function to determine if it needs upgrading. When we find -// one we are interested in, we then upgrade all calls to reflect the new -// function. -void llvm::UpgradeCallsToIntrinsic(Function* F) { +void llvm::UpgradeCallsToIntrinsic(Function *F) { assert(F && "Illegal attempt to upgrade a non-existent intrinsic."); - // Upgrade the function and check if it is a totaly new function. + // Check if this function should be upgraded and get the replacement function + // if there is one. Function *NewFn; if (UpgradeIntrinsicFunction(F, NewFn)) { - // Replace all uses to the old function with the new one if necessary. - for (Value::user_iterator UI = F->user_begin(), UE = F->user_end(); - UI != UE;) { + // Replace all users of the old function with the new function or new + // instructions. This is not a range loop because the call is deleted. + for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE; ) if (CallInst *CI = dyn_cast<CallInst>(*UI++)) UpgradeIntrinsicCall(CI, NewFn); - } + // Remove old function, no longer used, from the module. F->eraseFromParent(); } @@ -894,11 +1450,95 @@ bool llvm::UpgradeDebugInfo(Module &M) { return RetCode; } -void llvm::UpgradeMDStringConstant(std::string &String) { - const std::string OldPrefix = "llvm.vectorizer."; - if (String == "llvm.vectorizer.unroll") { - String = "llvm.loop.interleave.count"; - } else if (String.find(OldPrefix) == 0) { - String.replace(0, OldPrefix.size(), "llvm.loop.vectorize."); +bool llvm::UpgradeModuleFlags(Module &M) { + const NamedMDNode *ModFlags = M.getModuleFlagsMetadata(); + if (!ModFlags) + return false; + + bool HasObjCFlag = false, HasClassProperties = false; + for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) { + MDNode *Op = ModFlags->getOperand(I); + if (Op->getNumOperands() < 2) + continue; + MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1)); + if (!ID) + continue; + if (ID->getString() == "Objective-C Image Info Version") + HasObjCFlag = true; + if (ID->getString() == "Objective-C Class Properties") + HasClassProperties = true; } + // "Objective-C Class Properties" is recently added for Objective-C. We + // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module + // flag of value 0, so we can correclty report error when trying to link + // an ObjC bitcode without this module flag with an ObjC bitcode with this + // module flag. + if (HasObjCFlag && !HasClassProperties) { + M.addModuleFlag(llvm::Module::Error, "Objective-C Class Properties", + (uint32_t)0); + return true; + } + return false; +} + +static bool isOldLoopArgument(Metadata *MD) { + auto *T = dyn_cast_or_null<MDTuple>(MD); + if (!T) + return false; + if (T->getNumOperands() < 1) + return false; + auto *S = dyn_cast_or_null<MDString>(T->getOperand(0)); + if (!S) + return false; + return S->getString().startswith("llvm.vectorizer."); +} + +static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) { + StringRef OldPrefix = "llvm.vectorizer."; + assert(OldTag.startswith(OldPrefix) && "Expected old prefix"); + + if (OldTag == "llvm.vectorizer.unroll") + return MDString::get(C, "llvm.loop.interleave.count"); + + return MDString::get( + C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size())) + .str()); +} + +static Metadata *upgradeLoopArgument(Metadata *MD) { + auto *T = dyn_cast_or_null<MDTuple>(MD); + if (!T) + return MD; + if (T->getNumOperands() < 1) + return MD; + auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0)); + if (!OldTag) + return MD; + if (!OldTag->getString().startswith("llvm.vectorizer.")) + return MD; + + // This has an old tag. Upgrade it. + SmallVector<Metadata *, 8> Ops; + Ops.reserve(T->getNumOperands()); + Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString())); + for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I) + Ops.push_back(T->getOperand(I)); + + return MDTuple::get(T->getContext(), Ops); +} + +MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) { + auto *T = dyn_cast<MDTuple>(&N); + if (!T) + return &N; + + if (!llvm::any_of(T->operands(), isOldLoopArgument)) + return &N; + + SmallVector<Metadata *, 8> Ops; + Ops.reserve(T->getNumOperands()); + for (Metadata *MD : T->operands()) + Ops.push_back(upgradeLoopArgument(MD)); + + return MDTuple::get(T->getContext(), Ops); } diff --git a/lib/IR/BasicBlock.cpp b/lib/IR/BasicBlock.cpp index f61276fd436b4..4640b4f9d413c 100644 --- a/lib/IR/BasicBlock.cpp +++ b/lib/IR/BasicBlock.cpp @@ -162,6 +162,21 @@ CallInst *BasicBlock::getTerminatingMustTailCall() { return nullptr; } +CallInst *BasicBlock::getTerminatingDeoptimizeCall() { + if (InstList.empty()) + return nullptr; + auto *RI = dyn_cast<ReturnInst>(&InstList.back()); + if (!RI || RI == &InstList.front()) + return nullptr; + + if (auto *CI = dyn_cast_or_null<CallInst>(RI->getPrevNode())) + if (Function *F = CI->getCalledFunction()) + if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize) + return CI; + + return nullptr; +} + Instruction* BasicBlock::getFirstNonPHI() { for (Instruction &I : *this) if (!isa<PHINode>(I)) @@ -202,8 +217,8 @@ BasicBlock::iterator BasicBlock::getFirstInsertionPt() { } void BasicBlock::dropAllReferences() { - for(iterator I = begin(), E = end(); I != E; ++I) - I->dropAllReferences(); + for (Instruction &I : *this) + I.dropAllReferences(); } /// If this basic block has a single predecessor block, @@ -361,10 +376,8 @@ BasicBlock *BasicBlock::splitBasicBlock(iterator I, const Twine &BBName) { assert(I != InstList.end() && "Trying to get me to create degenerate basic block!"); - BasicBlock *InsertBefore = std::next(Function::iterator(this)) - .getNodePtrUnchecked(); - BasicBlock *New = BasicBlock::Create(getContext(), BBName, - getParent(), InsertBefore); + BasicBlock *New = BasicBlock::Create(getContext(), BBName, getParent(), + this->getNextNode()); // Save DebugLoc of split point before invalidating iterator. DebugLoc Loc = I->getDebugLoc(); diff --git a/lib/IR/CMakeLists.txt b/lib/IR/CMakeLists.txt index 40b4ec65e22be..07cec97084ee2 100644 --- a/lib/IR/CMakeLists.txt +++ b/lib/IR/CMakeLists.txt @@ -37,12 +37,14 @@ add_llvm_library(LLVMCore Mangler.cpp Metadata.cpp Module.cpp + ModuleSummaryIndex.cpp Operator.cpp + OptBisect.cpp Pass.cpp PassManager.cpp PassRegistry.cpp + ProfileSummary.cpp Statepoint.cpp - FunctionInfo.cpp Type.cpp TypeFinder.cpp Use.cpp @@ -56,12 +58,4 @@ add_llvm_library(LLVMCore ${LLVM_MAIN_INCLUDE_DIR}/llvm/IR ) -# PR24785: Workaround for hanging compilation. -if( MSVC_VERSION EQUAL 1800) - set_property( - SOURCE Function.cpp - PROPERTY COMPILE_FLAGS "/Og-" - ) -endif() - add_dependencies(LLVMCore intrinsics_gen) diff --git a/lib/IR/Comdat.cpp b/lib/IR/Comdat.cpp index 80715ff40ba92..fc1b48d1c190e 100644 --- a/lib/IR/Comdat.cpp +++ b/lib/IR/Comdat.cpp @@ -15,9 +15,6 @@ #include "llvm/ADT/StringMap.h" using namespace llvm; -Comdat::Comdat(SelectionKind SK, StringMapEntry<Comdat> *Name) - : Name(Name), SK(SK) {} - Comdat::Comdat(Comdat &&C) : Name(C.Name), SK(C.SK) {} Comdat::Comdat() : Name(nullptr), SK(Comdat::Any) {} diff --git a/lib/IR/ConstantFold.cpp b/lib/IR/ConstantFold.cpp index ce3fe03e2df7f..c06a99c053636 100644 --- a/lib/IR/ConstantFold.cpp +++ b/lib/IR/ConstantFold.cpp @@ -28,11 +28,9 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PatternMatch.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MathExtras.h" -#include <limits> using namespace llvm; using namespace llvm::PatternMatch; @@ -40,9 +38,9 @@ using namespace llvm::PatternMatch; // ConstantFold*Instruction Implementations //===----------------------------------------------------------------------===// -/// BitCastConstantVector - Convert the specified vector Constant node to the -/// specified vector type. At this point, we know that the elements of the -/// input vector constant are all simple integer or FP values. +/// Convert the specified vector Constant node to the specified vector type. +/// At this point, we know that the elements of the input vector constant are +/// all simple integer or FP values. static Constant *BitCastConstantVector(Constant *CV, VectorType *DstTy) { if (CV->isAllOnesValue()) return Constant::getAllOnesValue(DstTy); @@ -54,7 +52,7 @@ static Constant *BitCastConstantVector(Constant *CV, VectorType *DstTy) { unsigned NumElts = DstTy->getNumElements(); if (NumElts != CV->getType()->getVectorNumElements()) return nullptr; - + Type *DstEltTy = DstTy->getElementType(); SmallVector<Constant*, 16> Result; @@ -69,7 +67,7 @@ static Constant *BitCastConstantVector(Constant *CV, VectorType *DstTy) { return ConstantVector::get(Result); } -/// This function determines which opcode to use to fold two constant cast +/// This function determines which opcode to use to fold two constant cast /// expressions together. It uses CastInst::isEliminableCastPair to determine /// the opcode. Consequently its just a wrapper around that function. /// @brief Determine if it is valid to fold a cast of a cast @@ -120,7 +118,7 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { if (STy->getNumElements() == 0) break; ElTy = STy->getElementType(0); IdxList.push_back(Zero); - } else if (SequentialType *STy = + } else if (SequentialType *STy = dyn_cast<SequentialType>(ElTy)) { if (ElTy->isPointerTy()) break; // Can't index into pointers! ElTy = STy->getElementType(); @@ -136,7 +134,7 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { V, IdxList); } - // Handle casts from one vector constant to another. We know that the src + // Handle casts from one vector constant to another. We know that the src // and dest type have the same size (otherwise its an illegal cast). if (VectorType *DestPTy = dyn_cast<VectorType>(DestTy)) { if (VectorType *SrcTy = dyn_cast<VectorType>(V->getType())) { @@ -191,6 +189,10 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { if (FP->getType()->isPPC_FP128Ty()) return nullptr; + // Make sure dest type is compatible with the folded integer constant. + if (!DestTy->isIntegerTy()) + return nullptr; + return ConstantInt::get(FP->getContext(), FP->getValueAPF().bitcastToAPInt()); } @@ -199,15 +201,14 @@ static Constant *FoldBitCast(Constant *V, Type *DestTy) { } -/// ExtractConstantBytes - V is an integer constant which only has a subset of -/// its bytes used. The bytes used are indicated by ByteStart (which is the -/// first byte used, counting from the least significant byte) and ByteSize, -/// which is the number of bytes used. +/// V is an integer constant which only has a subset of its bytes used. +/// The bytes used are indicated by ByteStart (which is the first byte used, +/// counting from the least significant byte) and ByteSize, which is the number +/// of bytes used. /// /// This function analyzes the specified constant to see if the specified byte /// range can be returned as a simplified constant. If so, the constant is /// returned, otherwise null is returned. -/// static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, unsigned ByteSize) { assert(C->getType()->isIntegerTy() && @@ -217,7 +218,7 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, assert(ByteSize && "Must be accessing some piece"); assert(ByteStart+ByteSize <= CSize && "Extracting invalid piece from input"); assert(ByteSize != CSize && "Should not extract everything"); - + // Constant Integers are simple. if (ConstantInt *CI = dyn_cast<ConstantInt>(C)) { APInt V = CI->getValue(); @@ -226,7 +227,7 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, V = V.trunc(ByteSize*8); return ConstantInt::get(CI->getContext(), V); } - + // In the input is a constant expr, we might be able to recursively simplify. // If not, we definitely can't do anything. ConstantExpr *CE = dyn_cast<ConstantExpr>(C); @@ -238,12 +239,12 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, Constant *RHS = ExtractConstantBytes(CE->getOperand(1), ByteStart,ByteSize); if (!RHS) return nullptr; - + // X | -1 -> -1. if (ConstantInt *RHSC = dyn_cast<ConstantInt>(RHS)) if (RHSC->isAllOnesValue()) return RHSC; - + Constant *LHS = ExtractConstantBytes(CE->getOperand(0), ByteStart,ByteSize); if (!LHS) return nullptr; @@ -253,11 +254,11 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, Constant *RHS = ExtractConstantBytes(CE->getOperand(1), ByteStart,ByteSize); if (!RHS) return nullptr; - + // X & 0 -> 0. if (RHS->isNullValue()) return RHS; - + Constant *LHS = ExtractConstantBytes(CE->getOperand(0), ByteStart,ByteSize); if (!LHS) return nullptr; @@ -272,7 +273,7 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, if ((ShAmt & 7) != 0) return nullptr; ShAmt >>= 3; - + // If the extract is known to be all zeros, return zero. if (ByteStart >= CSize-ShAmt) return Constant::getNullValue(IntegerType::get(CE->getContext(), @@ -280,11 +281,11 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, // If the extract is known to be fully in the input, extract it. if (ByteStart+ByteSize+ShAmt <= CSize) return ExtractConstantBytes(CE->getOperand(0), ByteStart+ShAmt, ByteSize); - + // TODO: Handle the 'partially zero' case. return nullptr; } - + case Instruction::Shl: { ConstantInt *Amt = dyn_cast<ConstantInt>(CE->getOperand(1)); if (!Amt) @@ -294,7 +295,7 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, if ((ShAmt & 7) != 0) return nullptr; ShAmt >>= 3; - + // If the extract is known to be all zeros, return zero. if (ByteStart+ByteSize <= ShAmt) return Constant::getNullValue(IntegerType::get(CE->getContext(), @@ -302,15 +303,15 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, // If the extract is known to be fully in the input, extract it. if (ByteStart >= ShAmt) return ExtractConstantBytes(CE->getOperand(0), ByteStart-ShAmt, ByteSize); - + // TODO: Handle the 'partially zero' case. return nullptr; } - + case Instruction::ZExt: { unsigned SrcBitSize = cast<IntegerType>(CE->getOperand(0)->getType())->getBitWidth(); - + // If extracting something that is completely zero, return 0. if (ByteStart*8 >= SrcBitSize) return Constant::getNullValue(IntegerType::get(CE->getContext(), @@ -319,35 +320,34 @@ static Constant *ExtractConstantBytes(Constant *C, unsigned ByteStart, // If exactly extracting the input, return it. if (ByteStart == 0 && ByteSize*8 == SrcBitSize) return CE->getOperand(0); - + // If extracting something completely in the input, if if the input is a // multiple of 8 bits, recurse. if ((SrcBitSize&7) == 0 && (ByteStart+ByteSize)*8 <= SrcBitSize) return ExtractConstantBytes(CE->getOperand(0), ByteStart, ByteSize); - + // Otherwise, if extracting a subset of the input, which is not multiple of // 8 bits, do a shift and trunc to get the bits. if ((ByteStart+ByteSize)*8 < SrcBitSize) { assert((SrcBitSize&7) && "Shouldn't get byte sized case here"); Constant *Res = CE->getOperand(0); if (ByteStart) - Res = ConstantExpr::getLShr(Res, + Res = ConstantExpr::getLShr(Res, ConstantInt::get(Res->getType(), ByteStart*8)); return ConstantExpr::getTrunc(Res, IntegerType::get(C->getContext(), ByteSize*8)); } - + // TODO: Handle the 'partially zero' case. return nullptr; } } } -/// getFoldedSizeOf - Return a ConstantExpr with type DestTy for sizeof -/// on Ty, with any known factors factored out. If Folded is false, -/// return null if no factoring was possible, to avoid endlessly -/// bouncing an unfoldable expression back into the top-level folder. -/// +/// Return a ConstantExpr with type DestTy for sizeof on Ty, with any known +/// factors factored out. If Folded is false, return null if no factoring was +/// possible, to avoid endlessly bouncing an unfoldable expression back into the +/// top-level folder. static Constant *getFoldedSizeOf(Type *Ty, Type *DestTy, bool Folded) { if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { @@ -400,11 +400,10 @@ static Constant *getFoldedSizeOf(Type *Ty, Type *DestTy, return C; } -/// getFoldedAlignOf - Return a ConstantExpr with type DestTy for alignof -/// on Ty, with any known factors factored out. If Folded is false, -/// return null if no factoring was possible, to avoid endlessly -/// bouncing an unfoldable expression back into the top-level folder. -/// +/// Return a ConstantExpr with type DestTy for alignof on Ty, with any known +/// factors factored out. If Folded is false, return null if no factoring was +/// possible, to avoid endlessly bouncing an unfoldable expression back into the +/// top-level folder. static Constant *getFoldedAlignOf(Type *Ty, Type *DestTy, bool Folded) { // The alignment of an array is equal to the alignment of the @@ -466,11 +465,10 @@ static Constant *getFoldedAlignOf(Type *Ty, Type *DestTy, return C; } -/// getFoldedOffsetOf - Return a ConstantExpr with type DestTy for offsetof -/// on Ty and FieldNo, with any known factors factored out. If Folded is false, -/// return null if no factoring was possible, to avoid endlessly -/// bouncing an unfoldable expression back into the top-level folder. -/// +/// Return a ConstantExpr with type DestTy for offsetof on Ty and FieldNo, with +/// any known factors factored out. If Folded is false, return null if no +/// factoring was possible, to avoid endlessly bouncing an unfoldable expression +/// back into the top-level folder. static Constant *getFoldedOffsetOf(Type *Ty, Constant *FieldNo, Type *DestTy, bool Folded) { @@ -533,7 +531,8 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V, return UndefValue::get(DestTy); } - if (V->isNullValue() && !DestTy->isX86_MMXTy()) + if (V->isNullValue() && !DestTy->isX86_MMXTy() && + opc != Instruction::AddrSpaceCast) return Constant::getNullValue(DestTy); // If the cast operand is a constant expression, there's a few things we can @@ -600,12 +599,12 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V, return ConstantFP::get(V->getContext(), Val); } return nullptr; // Can't fold. - case Instruction::FPToUI: + case Instruction::FPToUI: case Instruction::FPToSI: if (ConstantFP *FPC = dyn_cast<ConstantFP>(V)) { const APFloat &V = FPC->getValueAPF(); bool ignored; - uint64_t x[2]; + uint64_t x[2]; uint32_t DestBitWidth = cast<IntegerType>(DestTy)->getBitWidth(); if (APFloat::opInvalidOp == V.convertToInteger(x, DestBitWidth, opc==Instruction::FPToSI, @@ -669,7 +668,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V, case Instruction::UIToFP: case Instruction::SIToFP: if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) { - APInt api = CI->getValue(); + const APInt &api = CI->getValue(); APFloat apf(DestTy->getFltSemantics(), APInt::getNullValue(DestTy->getPrimitiveSizeInBits())); if (APFloat::opOverflow & @@ -705,7 +704,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V, return ConstantInt::get(V->getContext(), CI->getValue().trunc(DestBitWidth)); } - + // The input must be a constantexpr. See if we can simplify this based on // the bytes we are demanding. Only do this if the source and dest are an // even multiple of a byte. @@ -713,7 +712,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V, (cast<IntegerType>(V->getType())->getBitWidth() & 7) == 0) if (Constant *Res = ExtractConstantBytes(V, 0, DestBitWidth / 8)) return Res; - + return nullptr; } case Instruction::BitCast: @@ -750,7 +749,7 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond, } Result.push_back(V); } - + // If we were able to build the vector, return it. if (Result.size() == V1->getType()->getVectorNumElements()) return ConstantVector::get(Result); @@ -814,12 +813,12 @@ Constant *llvm::ConstantFoldInsertElementInstruction(Constant *Val, Result.reserve(NumElts); auto *Ty = Type::getInt32Ty(Val->getContext()); uint64_t IdxVal = CIdx->getZExtValue(); - for (unsigned i = 0; i != NumElts; ++i) { + for (unsigned i = 0; i != NumElts; ++i) { if (i == IdxVal) { Result.push_back(Elt); continue; } - + Constant *C = ConstantExpr::getExtractElement(Val, ConstantInt::get(Ty, i)); Result.push_back(C); } @@ -839,7 +838,7 @@ Constant *llvm::ConstantFoldShuffleVectorInstruction(Constant *V1, // Don't break the bitcode reader hack. if (isa<ConstantExpr>(Mask)) return nullptr; - + unsigned SrcNumElts = V1->getType()->getVectorNumElements(); // Loop over the shuffle mask, evaluating each element. @@ -902,10 +901,10 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg, if (Idxs[0] == i) C = ConstantFoldInsertValueInstruction(C, Val, Idxs.slice(1)); - + Result.push_back(C); } - + if (StructType *ST = dyn_cast<StructType>(Agg->getType())) return ConstantStruct::get(ST, Result); if (ArrayType *AT = dyn_cast<ArrayType>(Agg->getType())) @@ -916,9 +915,11 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg, Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1, Constant *C2) { + assert(Instruction::isBinaryOp(Opcode) && "Non-binary instruction detected"); + // Handle UndefValue up front. if (isa<UndefValue>(C1) || isa<UndefValue>(C2)) { - switch (Opcode) { + switch (static_cast<Instruction::BinaryOps>(Opcode)) { case Instruction::Xor: if (isa<UndefValue>(C1) && isa<UndefValue>(C2)) // Handle undef ^ undef -> 0 special case. This is a common @@ -948,7 +949,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, case Instruction::SDiv: case Instruction::UDiv: // X / undef -> undef - if (match(C1, m_Zero())) + if (isa<UndefValue>(C2)) return C2; // undef / 0 -> undef // undef / 1 -> undef @@ -998,9 +999,22 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, return C1; // undef << X -> 0 return Constant::getNullValue(C1->getType()); + case Instruction::FAdd: + case Instruction::FSub: + case Instruction::FMul: + case Instruction::FDiv: + case Instruction::FRem: + // TODO: UNDEF handling for binary float instructions. + return nullptr; + case Instruction::BinaryOpsEnd: + llvm_unreachable("Invalid BinaryOp"); } } + // At this point neither constant should be an UndefValue. + assert(!isa<UndefValue>(C1) && !isa<UndefValue>(C2) && + "Unexpected UndefValue"); + // Handle simplifications when the RHS is a constant int. if (ConstantInt *CI2 = dyn_cast<ConstantInt>(C2)) { switch (Opcode) { @@ -1046,7 +1060,7 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, } // If and'ing the address of a global with a constant, fold it. - if (CE1->getOpcode() == Instruction::PtrToInt && + if (CE1->getOpcode() == Instruction::PtrToInt && isa<GlobalValue>(CE1->getOperand(0))) { GlobalValue *GV = cast<GlobalValue>(CE1->getOperand(0)); @@ -1102,7 +1116,6 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, return ConstantExpr::get(Opcode, C2, C1); } - // At this point we know neither constant is an UndefValue. if (ConstantInt *CI1 = dyn_cast<ConstantInt>(C1)) { if (ConstantInt *CI2 = dyn_cast<ConstantInt>(C2)) { const APInt &C1V = CI1->getValue(); @@ -1110,11 +1123,11 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, switch (Opcode) { default: break; - case Instruction::Add: + case Instruction::Add: return ConstantInt::get(CI1->getContext(), C1V + C2V); - case Instruction::Sub: + case Instruction::Sub: return ConstantInt::get(CI1->getContext(), C1V - C2V); - case Instruction::Mul: + case Instruction::Mul: return ConstantInt::get(CI1->getContext(), C1V * C2V); case Instruction::UDiv: assert(!CI2->isNullValue() && "Div by zero handled above"); @@ -1168,11 +1181,11 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, } } else if (ConstantFP *CFP1 = dyn_cast<ConstantFP>(C1)) { if (ConstantFP *CFP2 = dyn_cast<ConstantFP>(C2)) { - APFloat C1V = CFP1->getValueAPF(); - APFloat C2V = CFP2->getValueAPF(); + const APFloat &C1V = CFP1->getValueAPF(); + const APFloat &C2V = CFP2->getValueAPF(); APFloat C3V = C1V; // copy for modification switch (Opcode) { - default: + default: break; case Instruction::FAdd: (void)C3V.add(C2V, APFloat::rmNearestTiesToEven); @@ -1200,10 +1213,10 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, ConstantExpr::getExtractElement(C1, ConstantInt::get(Ty, i)); Constant *RHS = ConstantExpr::getExtractElement(C2, ConstantInt::get(Ty, i)); - + Result.push_back(ConstantExpr::get(Opcode, LHS, RHS)); } - + return ConstantVector::get(Result); } @@ -1259,8 +1272,8 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, return nullptr; } -/// isZeroSizedType - This type is zero sized if its an array or structure of -/// zero sized types. The only leaf zero sized type is an empty structure. +/// This type is zero-sized if it's an array or structure of zero-sized types. +/// The only leaf zero-sized type is an empty structure. static bool isMaybeZeroSizedType(Type *Ty) { if (StructType *STy = dyn_cast<StructType>(Ty)) { if (STy->isOpaque()) return true; // Can't say. @@ -1276,8 +1289,8 @@ static bool isMaybeZeroSizedType(Type *Ty) { return false; } -/// IdxCompare - Compare the two constants as though they were getelementptr -/// indices. This allows coercion of the types to be the same thing. +/// Compare the two constants as though they were getelementptr indices. +/// This allows coercion of the types to be the same thing. /// /// If the two constants are the "same" (after coercion), return 0. If the /// first is less than the second, return -1, if the second is less than the @@ -1316,11 +1329,11 @@ static int IdxCompare(Constant *C1, Constant *C2, Type *ElTy) { return 1; } -/// evaluateFCmpRelation - This function determines if there is anything we can -/// decide about the two constants provided. This doesn't need to handle simple -/// things like ConstantFP comparisons, but should instead handle ConstantExprs. -/// If we can determine that the two constants have a particular relation to -/// each other, we should return the corresponding FCmpInst predicate, +/// This function determines if there is anything we can decide about the two +/// constants provided. This doesn't need to handle simple things like +/// ConstantFP comparisons, but should instead handle ConstantExprs. +/// If we can determine that the two constants have a particular relation to +/// each other, we should return the corresponding FCmpInst predicate, /// otherwise return FCmpInst::BAD_FCMP_PREDICATE. This is used below in /// ConstantFoldCompareInstruction. /// @@ -1340,15 +1353,15 @@ static FCmpInst::Predicate evaluateFCmpRelation(Constant *V1, Constant *V2) { ConstantInt *R = nullptr; R = dyn_cast<ConstantInt>( ConstantExpr::getFCmp(FCmpInst::FCMP_OEQ, V1, V2)); - if (R && !R->isZero()) + if (R && !R->isZero()) return FCmpInst::FCMP_OEQ; R = dyn_cast<ConstantInt>( ConstantExpr::getFCmp(FCmpInst::FCMP_OLT, V1, V2)); - if (R && !R->isZero()) + if (R && !R->isZero()) return FCmpInst::FCMP_OLT; R = dyn_cast<ConstantInt>( ConstantExpr::getFCmp(FCmpInst::FCMP_OGT, V1, V2)); - if (R && !R->isZero()) + if (R && !R->isZero()) return FCmpInst::FCMP_OGT; // Nothing more we can do @@ -1403,12 +1416,12 @@ static ICmpInst::Predicate areGlobalsPotentiallyEqual(const GlobalValue *GV1, return ICmpInst::BAD_ICMP_PREDICATE; } -/// evaluateICmpRelation - This function determines if there is anything we can -/// decide about the two constants provided. This doesn't need to handle simple -/// things like integer comparisons, but should instead handle ConstantExprs -/// and GlobalValues. If we can determine that the two constants have a -/// particular relation to each other, we should return the corresponding ICmp -/// predicate, otherwise return ICmpInst::BAD_ICMP_PREDICATE. +/// This function determines if there is anything we can decide about the two +/// constants provided. This doesn't need to handle simple things like integer +/// comparisons, but should instead handle ConstantExprs and GlobalValues. +/// If we can determine that the two constants have a particular relation to +/// each other, we should return the corresponding ICmp predicate, otherwise +/// return ICmpInst::BAD_ICMP_PREDICATE. /// /// To simplify this code we canonicalize the relation so that the first /// operand is always the most "complex" of the two. We consider simple @@ -1430,7 +1443,7 @@ static ICmpInst::Predicate evaluateICmpRelation(Constant *V1, Constant *V2, ConstantInt *R = nullptr; ICmpInst::Predicate pred = ICmpInst::ICMP_EQ; R = dyn_cast<ConstantInt>(ConstantExpr::getICmp(pred, V1, V2)); - if (R && !R->isZero()) + if (R && !R->isZero()) return pred; pred = isSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; R = dyn_cast<ConstantInt>(ConstantExpr::getICmp(pred, V1, V2)); @@ -1446,14 +1459,14 @@ static ICmpInst::Predicate evaluateICmpRelation(Constant *V1, Constant *V2, } // If the first operand is simple, swap operands. - ICmpInst::Predicate SwappedRelation = + ICmpInst::Predicate SwappedRelation = evaluateICmpRelation(V2, V1, isSigned); if (SwappedRelation != ICmpInst::BAD_ICMP_PREDICATE) return ICmpInst::getSwappedPredicate(SwappedRelation); } else if (const GlobalValue *GV = dyn_cast<GlobalValue>(V1)) { if (isa<ConstantExpr>(V2)) { // Swap as necessary. - ICmpInst::Predicate SwappedRelation = + ICmpInst::Predicate SwappedRelation = evaluateICmpRelation(V2, V1, isSigned); if (SwappedRelation != ICmpInst::BAD_ICMP_PREDICATE) return ICmpInst::getSwappedPredicate(SwappedRelation); @@ -1476,13 +1489,13 @@ static ICmpInst::Predicate evaluateICmpRelation(Constant *V1, Constant *V2, } } else if (const BlockAddress *BA = dyn_cast<BlockAddress>(V1)) { if (isa<ConstantExpr>(V2)) { // Swap as necessary. - ICmpInst::Predicate SwappedRelation = + ICmpInst::Predicate SwappedRelation = evaluateICmpRelation(V2, V1, isSigned); if (SwappedRelation != ICmpInst::BAD_ICMP_PREDICATE) return ICmpInst::getSwappedPredicate(SwappedRelation); return ICmpInst::BAD_ICMP_PREDICATE; } - + // Now we know that the RHS is a GlobalValue, BlockAddress or simple // constant (which, since the types must match, means that it is a // ConstantPointerNull). @@ -1517,6 +1530,10 @@ static ICmpInst::Predicate evaluateICmpRelation(Constant *V1, Constant *V2, case Instruction::BitCast: case Instruction::ZExt: case Instruction::SExt: + // We can't evaluate floating point casts or truncations. + if (CE1Op0->getType()->isFloatingPointTy()) + break; + // If the cast is not actually changing bits, and the second operand is a // null pointer, do the comparison with the pre-casted value. if (V2->isNullValue() && @@ -1524,7 +1541,7 @@ static ICmpInst::Predicate evaluateICmpRelation(Constant *V1, Constant *V2, if (CE1->getOpcode() == Instruction::ZExt) isSigned = false; if (CE1->getOpcode() == Instruction::SExt) isSigned = true; return evaluateICmpRelation(CE1Op0, - Constant::getNullValue(CE1Op0->getType()), + Constant::getNullValue(CE1Op0->getType()), isSigned); } break; @@ -1541,7 +1558,7 @@ static ICmpInst::Predicate evaluateICmpRelation(Constant *V1, Constant *V2, // Weak linkage GVals could be zero or not. We're comparing that // to null pointer so its greater-or-equal return isSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; - else + else // If its not weak linkage, the GVal must have a non-zero address // so the result is greater-than return isSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; @@ -1656,7 +1673,7 @@ static ICmpInst::Predicate evaluateICmpRelation(Constant *V1, Constant *V2, return ICmpInst::BAD_ICMP_PREDICATE; } -Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, +Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, Constant *C1, Constant *C2) { Type *ResultTy; if (VectorType *VT = dyn_cast<VectorType>(C1->getType())) @@ -1729,8 +1746,8 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, } if (isa<ConstantInt>(C1) && isa<ConstantInt>(C2)) { - APInt V1 = cast<ConstantInt>(C1)->getValue(); - APInt V2 = cast<ConstantInt>(C2)->getValue(); + const APInt &V1 = cast<ConstantInt>(C1)->getValue(); + const APInt &V2 = cast<ConstantInt>(C2)->getValue(); switch (pred) { default: llvm_unreachable("Invalid ICmp Predicate"); case ICmpInst::ICMP_EQ: return ConstantInt::get(ResultTy, V1 == V2); @@ -1745,8 +1762,8 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, case ICmpInst::ICMP_UGE: return ConstantInt::get(ResultTy, V1.uge(V2)); } } else if (isa<ConstantFP>(C1) && isa<ConstantFP>(C2)) { - APFloat C1V = cast<ConstantFP>(C1)->getValueAPF(); - APFloat C2V = cast<ConstantFP>(C2)->getValueAPF(); + const APFloat &C1V = cast<ConstantFP>(C1)->getValueAPF(); + const APFloat &C2V = cast<ConstantFP>(C2)->getValueAPF(); APFloat::cmpResult R = C1V.compare(C2V); switch (pred) { default: llvm_unreachable("Invalid FCmp Predicate"); @@ -1759,17 +1776,17 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, case FCmpInst::FCMP_UEQ: return ConstantInt::get(ResultTy, R==APFloat::cmpUnordered || R==APFloat::cmpEqual); - case FCmpInst::FCMP_OEQ: + case FCmpInst::FCMP_OEQ: return ConstantInt::get(ResultTy, R==APFloat::cmpEqual); case FCmpInst::FCMP_UNE: return ConstantInt::get(ResultTy, R!=APFloat::cmpEqual); - case FCmpInst::FCMP_ONE: + case FCmpInst::FCMP_ONE: return ConstantInt::get(ResultTy, R==APFloat::cmpLessThan || R==APFloat::cmpGreaterThan); - case FCmpInst::FCMP_ULT: + case FCmpInst::FCMP_ULT: return ConstantInt::get(ResultTy, R==APFloat::cmpUnordered || R==APFloat::cmpLessThan); - case FCmpInst::FCMP_OLT: + case FCmpInst::FCMP_OLT: return ConstantInt::get(ResultTy, R==APFloat::cmpLessThan); case FCmpInst::FCMP_UGT: return ConstantInt::get(ResultTy, R==APFloat::cmpUnordered || @@ -1778,12 +1795,12 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, return ConstantInt::get(ResultTy, R==APFloat::cmpGreaterThan); case FCmpInst::FCMP_ULE: return ConstantInt::get(ResultTy, R!=APFloat::cmpGreaterThan); - case FCmpInst::FCMP_OLE: + case FCmpInst::FCMP_OLE: return ConstantInt::get(ResultTy, R==APFloat::cmpLessThan || R==APFloat::cmpEqual); case FCmpInst::FCMP_UGE: return ConstantInt::get(ResultTy, R!=APFloat::cmpLessThan); - case FCmpInst::FCMP_OGE: + case FCmpInst::FCMP_OGE: return ConstantInt::get(ResultTy, R==APFloat::cmpGreaterThan || R==APFloat::cmpEqual); } @@ -1798,10 +1815,10 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, ConstantExpr::getExtractElement(C1, ConstantInt::get(Ty, i)); Constant *C2E = ConstantExpr::getExtractElement(C2, ConstantInt::get(Ty, i)); - + ResElts.push_back(ConstantExpr::getCompare(pred, C1E, C2E)); } - + return ConstantVector::get(ResElts); } @@ -1841,23 +1858,23 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, break; case FCmpInst::FCMP_OLE: // We know that C1 <= C2 // We can only partially decide this relation. - if (pred == FCmpInst::FCMP_UGT || pred == FCmpInst::FCMP_OGT) + if (pred == FCmpInst::FCMP_UGT || pred == FCmpInst::FCMP_OGT) Result = 0; - else if (pred == FCmpInst::FCMP_ULT || pred == FCmpInst::FCMP_OLT) + else if (pred == FCmpInst::FCMP_ULT || pred == FCmpInst::FCMP_OLT) Result = 1; break; case FCmpInst::FCMP_OGE: // We known that C1 >= C2 // We can only partially decide this relation. - if (pred == FCmpInst::FCMP_ULT || pred == FCmpInst::FCMP_OLT) + if (pred == FCmpInst::FCMP_ULT || pred == FCmpInst::FCMP_OLT) Result = 0; - else if (pred == FCmpInst::FCMP_UGT || pred == FCmpInst::FCMP_OGT) + else if (pred == FCmpInst::FCMP_UGT || pred == FCmpInst::FCMP_OGT) Result = 1; break; case FCmpInst::FCMP_ONE: // We know that C1 != C2 // We can only partially decide this relation. - if (pred == FCmpInst::FCMP_OEQ || pred == FCmpInst::FCMP_UEQ) + if (pred == FCmpInst::FCMP_OEQ || pred == FCmpInst::FCMP_UEQ) Result = 0; - else if (pred == FCmpInst::FCMP_ONE || pred == FCmpInst::FCMP_UNE) + else if (pred == FCmpInst::FCMP_ONE || pred == FCmpInst::FCMP_UNE) Result = 1; break; } @@ -1979,8 +1996,7 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred, return nullptr; } -/// isInBoundsIndices - Test whether the given sequence of *normalized* indices -/// is "inbounds". +/// Test whether the given sequence of *normalized* indices is "inbounds". template<typename IndexTy> static bool isInBoundsIndices(ArrayRef<IndexTy> Idxs) { // No indices means nothing that could be out of bounds. @@ -1999,7 +2015,7 @@ static bool isInBoundsIndices(ArrayRef<IndexTy> Idxs) { return true; } -/// \brief Test whether a given ConstantInt is in-range for a SequentialType. +/// Test whether a given ConstantInt is in-range for a SequentialType. static bool isIndexInRangeOfSequentialType(SequentialType *STy, const ConstantInt *CI) { // And indices are valid when indexing along a pointer @@ -2040,11 +2056,13 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, return C; if (isa<UndefValue>(C)) { - PointerType *Ptr = cast<PointerType>(C->getType()); - Type *Ty = GetElementPtrInst::getIndexedType( - cast<PointerType>(Ptr->getScalarType())->getElementType(), Idxs); + PointerType *PtrTy = cast<PointerType>(C->getType()->getScalarType()); + Type *Ty = GetElementPtrInst::getIndexedType(PointeeTy, Idxs); assert(Ty && "Invalid indices for GEP!"); - return UndefValue::get(PointerType::get(Ty, Ptr->getAddressSpace())); + Type *GEPTy = PointerType::get(Ty, PtrTy->getAddressSpace()); + if (VectorType *VT = dyn_cast<VectorType>(C->getType())) + GEPTy = VectorType::get(GEPTy, VT->getNumElements()); + return UndefValue::get(GEPTy); } if (C->isNullValue()) { @@ -2055,12 +2073,14 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, break; } if (isNull) { - PointerType *Ptr = cast<PointerType>(C->getType()); - Type *Ty = GetElementPtrInst::getIndexedType( - cast<PointerType>(Ptr->getScalarType())->getElementType(), Idxs); + PointerType *PtrTy = cast<PointerType>(C->getType()->getScalarType()); + Type *Ty = GetElementPtrInst::getIndexedType(PointeeTy, Idxs); + assert(Ty && "Invalid indices for GEP!"); - return ConstantPointerNull::get(PointerType::get(Ty, - Ptr->getAddressSpace())); + Type *GEPTy = PointerType::get(Ty, PtrTy->getAddressSpace()); + if (VectorType *VT = dyn_cast<VectorType>(C->getType())) + GEPTy = VectorType::get(GEPTy, VT->getNumElements()); + return Constant::getNullValue(GEPTy); } } @@ -2172,52 +2192,68 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, bool Unknown = !isa<ConstantInt>(Idxs[0]); for (unsigned i = 1, e = Idxs.size(); i != e; Prev = Ty, Ty = cast<CompositeType>(Ty)->getTypeAtIndex(Idxs[i]), ++i) { - if (ConstantInt *CI = dyn_cast<ConstantInt>(Idxs[i])) { - if (isa<ArrayType>(Ty) || isa<VectorType>(Ty)) - if (CI->getSExtValue() > 0 && - !isIndexInRangeOfSequentialType(cast<SequentialType>(Ty), CI)) { - if (isa<SequentialType>(Prev)) { - // It's out of range, but we can factor it into the prior - // dimension. - NewIdxs.resize(Idxs.size()); - uint64_t NumElements = 0; - if (auto *ATy = dyn_cast<ArrayType>(Ty)) - NumElements = ATy->getNumElements(); - else - NumElements = cast<VectorType>(Ty)->getNumElements(); - - ConstantInt *Factor = ConstantInt::get(CI->getType(), NumElements); - NewIdxs[i] = ConstantExpr::getSRem(CI, Factor); - - Constant *PrevIdx = cast<Constant>(Idxs[i-1]); - Constant *Div = ConstantExpr::getSDiv(CI, Factor); - - unsigned CommonExtendedWidth = - std::max(PrevIdx->getType()->getIntegerBitWidth(), - Div->getType()->getIntegerBitWidth()); - CommonExtendedWidth = std::max(CommonExtendedWidth, 64U); - - // Before adding, extend both operands to i64 to avoid - // overflow trouble. - if (!PrevIdx->getType()->isIntegerTy(CommonExtendedWidth)) - PrevIdx = ConstantExpr::getSExt( - PrevIdx, - Type::getIntNTy(Div->getContext(), CommonExtendedWidth)); - if (!Div->getType()->isIntegerTy(CommonExtendedWidth)) - Div = ConstantExpr::getSExt( - Div, Type::getIntNTy(Div->getContext(), CommonExtendedWidth)); - - NewIdxs[i-1] = ConstantExpr::getAdd(PrevIdx, Div); - } else { - // It's out of range, but the prior dimension is a struct - // so we can't do anything about it. - Unknown = true; - } - } - } else { + auto *CI = dyn_cast<ConstantInt>(Idxs[i]); + if (!CI) { // We don't know if it's in range or not. Unknown = true; + continue; + } + if (isa<StructType>(Ty)) { + // The verify makes sure that GEPs into a struct are in range. + continue; } + auto *STy = cast<SequentialType>(Ty); + if (isa<PointerType>(STy)) { + // We don't know if it's in range or not. + Unknown = true; + continue; + } + if (isa<VectorType>(STy)) { + // There can be awkward padding in after a non-power of two vector. + Unknown = true; + continue; + } + if (isIndexInRangeOfSequentialType(STy, CI)) + // It's in range, skip to the next index. + continue; + if (!isa<SequentialType>(Prev)) { + // It's out of range, but the prior dimension is a struct + // so we can't do anything about it. + Unknown = true; + continue; + } + if (CI->getSExtValue() < 0) { + // It's out of range and negative, don't try to factor it. + Unknown = true; + continue; + } + // It's out of range, but we can factor it into the prior + // dimension. + NewIdxs.resize(Idxs.size()); + // Determine the number of elements in our sequential type. + uint64_t NumElements = STy->getArrayNumElements(); + + ConstantInt *Factor = ConstantInt::get(CI->getType(), NumElements); + NewIdxs[i] = ConstantExpr::getSRem(CI, Factor); + + Constant *PrevIdx = cast<Constant>(Idxs[i - 1]); + Constant *Div = ConstantExpr::getSDiv(CI, Factor); + + unsigned CommonExtendedWidth = + std::max(PrevIdx->getType()->getIntegerBitWidth(), + Div->getType()->getIntegerBitWidth()); + CommonExtendedWidth = std::max(CommonExtendedWidth, 64U); + + // Before adding, extend both operands to i64 to avoid + // overflow trouble. + if (!PrevIdx->getType()->isIntegerTy(CommonExtendedWidth)) + PrevIdx = ConstantExpr::getSExt( + PrevIdx, Type::getIntNTy(Div->getContext(), CommonExtendedWidth)); + if (!Div->getType()->isIntegerTy(CommonExtendedWidth)) + Div = ConstantExpr::getSExt( + Div, Type::getIntNTy(Div->getContext(), CommonExtendedWidth)); + + NewIdxs[i - 1] = ConstantExpr::getAdd(PrevIdx, Div); } // If we did any factoring, start over with the adjusted indices. @@ -2237,22 +2273,6 @@ static Constant *ConstantFoldGetElementPtrImpl(Type *PointeeTy, Constant *C, return nullptr; } -Constant *llvm::ConstantFoldGetElementPtr(Constant *C, - bool inBounds, - ArrayRef<Constant *> Idxs) { - return ConstantFoldGetElementPtrImpl( - cast<PointerType>(C->getType()->getScalarType())->getElementType(), C, - inBounds, Idxs); -} - -Constant *llvm::ConstantFoldGetElementPtr(Constant *C, - bool inBounds, - ArrayRef<Value *> Idxs) { - return ConstantFoldGetElementPtrImpl( - cast<PointerType>(C->getType()->getScalarType())->getElementType(), C, - inBounds, Idxs); -} - Constant *llvm::ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool inBounds, ArrayRef<Constant *> Idxs) { diff --git a/lib/IR/ConstantFold.h b/lib/IR/ConstantFold.h index 42a9c6ba908a8..9b0a937c84df1 100644 --- a/lib/IR/ConstantFold.h +++ b/lib/IR/ConstantFold.h @@ -19,9 +19,8 @@ #ifndef LLVM_LIB_IR_CONSTANTFOLD_H #define LLVM_LIB_IR_CONSTANTFOLD_H -#include "llvm/ADT/ArrayRef.h" - namespace llvm { +template <typename T> class ArrayRef; class Value; class Constant; class Type; @@ -45,12 +44,8 @@ namespace llvm { ArrayRef<unsigned> Idxs); Constant *ConstantFoldBinaryInstruction(unsigned Opcode, Constant *V1, Constant *V2); - Constant *ConstantFoldCompareInstruction(unsigned short predicate, + Constant *ConstantFoldCompareInstruction(unsigned short predicate, Constant *C1, Constant *C2); - Constant *ConstantFoldGetElementPtr(Constant *C, bool inBounds, - ArrayRef<Constant *> Idxs); - Constant *ConstantFoldGetElementPtr(Constant *C, bool inBounds, - ArrayRef<Value *> Idxs); Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool inBounds, ArrayRef<Constant *> Idxs); Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool inBounds, diff --git a/lib/IR/ConstantRange.cpp b/lib/IR/ConstantRange.cpp index 48f9b27a25ae7..0f5c7128f3d09 100644 --- a/lib/IR/ConstantRange.cpp +++ b/lib/IR/ConstantRange.cpp @@ -60,60 +60,60 @@ ConstantRange ConstantRange::makeAllowedICmpRegion(CmpInst::Predicate Pred, switch (Pred) { default: llvm_unreachable("Invalid ICmp predicate to makeAllowedICmpRegion()"); - case CmpInst::ICMP_EQ: - return CR; - case CmpInst::ICMP_NE: - if (CR.isSingleElement()) - return ConstantRange(CR.getUpper(), CR.getLower()); + case CmpInst::ICMP_EQ: + return CR; + case CmpInst::ICMP_NE: + if (CR.isSingleElement()) + return ConstantRange(CR.getUpper(), CR.getLower()); + return ConstantRange(W); + case CmpInst::ICMP_ULT: { + APInt UMax(CR.getUnsignedMax()); + if (UMax.isMinValue()) + return ConstantRange(W, /* empty */ false); + return ConstantRange(APInt::getMinValue(W), UMax); + } + case CmpInst::ICMP_SLT: { + APInt SMax(CR.getSignedMax()); + if (SMax.isMinSignedValue()) + return ConstantRange(W, /* empty */ false); + return ConstantRange(APInt::getSignedMinValue(W), SMax); + } + case CmpInst::ICMP_ULE: { + APInt UMax(CR.getUnsignedMax()); + if (UMax.isMaxValue()) return ConstantRange(W); - case CmpInst::ICMP_ULT: { - APInt UMax(CR.getUnsignedMax()); - if (UMax.isMinValue()) - return ConstantRange(W, /* empty */ false); - return ConstantRange(APInt::getMinValue(W), UMax); - } - case CmpInst::ICMP_SLT: { - APInt SMax(CR.getSignedMax()); - if (SMax.isMinSignedValue()) - return ConstantRange(W, /* empty */ false); - return ConstantRange(APInt::getSignedMinValue(W), SMax); - } - case CmpInst::ICMP_ULE: { - APInt UMax(CR.getUnsignedMax()); - if (UMax.isMaxValue()) - return ConstantRange(W); - return ConstantRange(APInt::getMinValue(W), UMax + 1); - } - case CmpInst::ICMP_SLE: { - APInt SMax(CR.getSignedMax()); - if (SMax.isMaxSignedValue()) - return ConstantRange(W); - return ConstantRange(APInt::getSignedMinValue(W), SMax + 1); - } - case CmpInst::ICMP_UGT: { - APInt UMin(CR.getUnsignedMin()); - if (UMin.isMaxValue()) - return ConstantRange(W, /* empty */ false); - return ConstantRange(UMin + 1, APInt::getNullValue(W)); - } - case CmpInst::ICMP_SGT: { - APInt SMin(CR.getSignedMin()); - if (SMin.isMaxSignedValue()) - return ConstantRange(W, /* empty */ false); - return ConstantRange(SMin + 1, APInt::getSignedMinValue(W)); - } - case CmpInst::ICMP_UGE: { - APInt UMin(CR.getUnsignedMin()); - if (UMin.isMinValue()) - return ConstantRange(W); - return ConstantRange(UMin, APInt::getNullValue(W)); - } - case CmpInst::ICMP_SGE: { - APInt SMin(CR.getSignedMin()); - if (SMin.isMinSignedValue()) - return ConstantRange(W); - return ConstantRange(SMin, APInt::getSignedMinValue(W)); - } + return ConstantRange(APInt::getMinValue(W), UMax + 1); + } + case CmpInst::ICMP_SLE: { + APInt SMax(CR.getSignedMax()); + if (SMax.isMaxSignedValue()) + return ConstantRange(W); + return ConstantRange(APInt::getSignedMinValue(W), SMax + 1); + } + case CmpInst::ICMP_UGT: { + APInt UMin(CR.getUnsignedMin()); + if (UMin.isMaxValue()) + return ConstantRange(W, /* empty */ false); + return ConstantRange(UMin + 1, APInt::getNullValue(W)); + } + case CmpInst::ICMP_SGT: { + APInt SMin(CR.getSignedMin()); + if (SMin.isMaxSignedValue()) + return ConstantRange(W, /* empty */ false); + return ConstantRange(SMin + 1, APInt::getSignedMinValue(W)); + } + case CmpInst::ICMP_UGE: { + APInt UMin(CR.getUnsignedMin()); + if (UMin.isMinValue()) + return ConstantRange(W); + return ConstantRange(UMin, APInt::getNullValue(W)); + } + case CmpInst::ICMP_SGE: { + APInt SMin(CR.getSignedMin()); + if (SMin.isMinSignedValue()) + return ConstantRange(W); + return ConstantRange(SMin, APInt::getSignedMinValue(W)); + } } } @@ -127,9 +127,48 @@ ConstantRange ConstantRange::makeSatisfyingICmpRegion(CmpInst::Predicate Pred, .inverse(); } -ConstantRange ConstantRange::makeNoWrapRegion(Instruction::BinaryOps BinOp, - const APInt &C, - unsigned NoWrapKind) { +ConstantRange ConstantRange::makeExactICmpRegion(CmpInst::Predicate Pred, + const APInt &C) { + // Computes the exact range that is equal to both the constant ranges returned + // by makeAllowedICmpRegion and makeSatisfyingICmpRegion. This is always true + // when RHS is a singleton such as an APInt and so the assert is valid. + // However for non-singleton RHS, for example ult [2,5) makeAllowedICmpRegion + // returns [0,4) but makeSatisfyICmpRegion returns [0,2). + // + assert(makeAllowedICmpRegion(Pred, C) == makeSatisfyingICmpRegion(Pred, C)); + return makeAllowedICmpRegion(Pred, C); +} + +bool ConstantRange::getEquivalentICmp(CmpInst::Predicate &Pred, + APInt &RHS) const { + bool Success = false; + + if (isFullSet() || isEmptySet()) { + Pred = isEmptySet() ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE; + RHS = APInt(getBitWidth(), 0); + Success = true; + } else if (getLower().isMinSignedValue() || getLower().isMinValue()) { + Pred = + getLower().isMinSignedValue() ? CmpInst::ICMP_SLT : CmpInst::ICMP_ULT; + RHS = getUpper(); + Success = true; + } else if (getUpper().isMinSignedValue() || getUpper().isMinValue()) { + Pred = + getUpper().isMinSignedValue() ? CmpInst::ICMP_SGE : CmpInst::ICMP_UGE; + RHS = getLower(); + Success = true; + } + + assert((!Success || ConstantRange::makeExactICmpRegion(Pred, RHS) == *this) && + "Bad result!"); + + return Success; +} + +ConstantRange +ConstantRange::makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp, + const ConstantRange &Other, + unsigned NoWrapKind) { typedef OverflowingBinaryOperator OBO; // Computes the intersection of CR0 and CR1. It is different from @@ -149,29 +188,36 @@ ConstantRange ConstantRange::makeNoWrapRegion(Instruction::BinaryOps BinOp, NoWrapKind == (OBO::NoUnsignedWrap | OBO::NoSignedWrap)) && "NoWrapKind invalid!"); - unsigned BitWidth = C.getBitWidth(); + unsigned BitWidth = Other.getBitWidth(); if (BinOp != Instruction::Add) // Conservative answer: empty set return ConstantRange(BitWidth, false); - if (C.isMinValue()) - // Full set: nothing signed / unsigned wraps when added to 0. - return ConstantRange(BitWidth); + if (auto *C = Other.getSingleElement()) + if (C->isMinValue()) + // Full set: nothing signed / unsigned wraps when added to 0. + return ConstantRange(BitWidth); ConstantRange Result(BitWidth); if (NoWrapKind & OBO::NoUnsignedWrap) - Result = SubsetIntersect(Result, - ConstantRange(APInt::getNullValue(BitWidth), -C)); + Result = + SubsetIntersect(Result, ConstantRange(APInt::getNullValue(BitWidth), + -Other.getUnsignedMax())); if (NoWrapKind & OBO::NoSignedWrap) { - if (C.isStrictlyPositive()) + APInt SignedMin = Other.getSignedMin(); + APInt SignedMax = Other.getSignedMax(); + + if (SignedMax.isStrictlyPositive()) Result = SubsetIntersect( - Result, ConstantRange(APInt::getSignedMinValue(BitWidth), - APInt::getSignedMinValue(BitWidth) - C)); - else + Result, + ConstantRange(APInt::getSignedMinValue(BitWidth), + APInt::getSignedMinValue(BitWidth) - SignedMax)); + + if (SignedMin.isNegative()) Result = SubsetIntersect( - Result, ConstantRange(APInt::getSignedMinValue(BitWidth) - C, + Result, ConstantRange(APInt::getSignedMinValue(BitWidth) - SignedMin, APInt::getSignedMinValue(BitWidth))); } @@ -544,7 +590,7 @@ ConstantRange ConstantRange::truncate(uint32_t DstTySize) const { // We use the non-wrapped set code to analyze the [Lower, MaxValue) part, and // then we do the union with [MaxValue, Upper) if (isWrappedSet()) { - // if Upper is greater than Max Value, it covers the whole truncated range. + // If Upper is greater than Max Value, it covers the whole truncated range. if (Upper.uge(MaxValue)) return ConstantRange(DstTySize, /*isFullSet=*/true); @@ -568,7 +614,7 @@ ConstantRange ConstantRange::truncate(uint32_t DstTySize) const { return ConstantRange(LowerDiv.trunc(DstTySize), UpperDiv.trunc(DstTySize)).unionWith(Union); - // The truncated value wrapps around. Check if we can do better than fullset. + // The truncated value wraps around. Check if we can do better than fullset. APInt UpperModulo = UpperDiv - MaxBitValue; if (UpperModulo.ult(LowerDiv)) return ConstantRange(LowerDiv.trunc(DstTySize), @@ -667,6 +713,13 @@ ConstantRange::multiply(const ConstantRange &Other) const { this_max * Other_max + 1); ConstantRange UR = Result_zext.truncate(getBitWidth()); + // If the unsigned range doesn't wrap, and isn't negative then it's a range + // from one positive number to another which is as good as we can generate. + // In this case, skip the extra work of generating signed ranges which aren't + // going to be better than this range. + if (!UR.isWrappedSet() && UR.getLower().isNonNegative()) + return UR; + // Now the signed range. Because we could be dealing with negative numbers // here, the lower bound is the smallest of the cartesian product of the // lower and upper ranges; for example: @@ -714,6 +767,32 @@ ConstantRange::umax(const ConstantRange &Other) const { } ConstantRange +ConstantRange::smin(const ConstantRange &Other) const { + // X smin Y is: range(smin(X_smin, Y_smin), + // smin(X_smax, Y_smax)) + if (isEmptySet() || Other.isEmptySet()) + return ConstantRange(getBitWidth(), /*isFullSet=*/false); + APInt NewL = APIntOps::smin(getSignedMin(), Other.getSignedMin()); + APInt NewU = APIntOps::smin(getSignedMax(), Other.getSignedMax()) + 1; + if (NewU == NewL) + return ConstantRange(getBitWidth(), /*isFullSet=*/true); + return ConstantRange(NewL, NewU); +} + +ConstantRange +ConstantRange::umin(const ConstantRange &Other) const { + // X umin Y is: range(umin(X_umin, Y_umin), + // umin(X_umax, Y_umax)) + if (isEmptySet() || Other.isEmptySet()) + return ConstantRange(getBitWidth(), /*isFullSet=*/false); + APInt NewL = APIntOps::umin(getUnsignedMin(), Other.getUnsignedMin()); + APInt NewU = APIntOps::umin(getUnsignedMax(), Other.getUnsignedMax()) + 1; + if (NewU == NewL) + return ConstantRange(getBitWidth(), /*isFullSet=*/true); + return ConstantRange(NewL, NewU); +} + +ConstantRange ConstantRange::udiv(const ConstantRange &RHS) const { if (isEmptySet() || RHS.isEmptySet() || RHS.getUnsignedMax() == 0) return ConstantRange(getBitWidth(), /*isFullSet=*/false); @@ -819,6 +898,6 @@ void ConstantRange::print(raw_ostream &OS) const { /// dump - Allow printing from a debugger easily... /// -void ConstantRange::dump() const { +LLVM_DUMP_METHOD void ConstantRange::dump() const { print(dbgs()); } diff --git a/lib/IR/Constants.cpp b/lib/IR/Constants.cpp index 0898bf6453851..d8d55b472f32c 100644 --- a/lib/IR/Constants.cpp +++ b/lib/IR/Constants.cpp @@ -14,8 +14,6 @@ #include "llvm/IR/Constants.h" #include "ConstantFold.h" #include "LLVMContextImpl.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" @@ -26,7 +24,6 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" @@ -42,6 +39,8 @@ using namespace llvm; void Constant::anchor() { } +void ConstantData::anchor() {} + bool Constant::isNegativeZeroValue() const { // Floating point values have an explicit -0.0 value. if (const ConstantFP *CFP = dyn_cast<ConstantFP>(this)) @@ -192,7 +191,7 @@ bool Constant::isNotMinSignedValue() const { return false; } -// Constructor to create a '0' constant of arbitrary type... +/// Constructor to create a '0' constant of arbitrary type. Constant *Constant::getNullValue(Type *Ty) { switch (Ty->getTypeID()) { case Type::IntegerTyID: @@ -263,19 +262,9 @@ Constant *Constant::getAllOnesValue(Type *Ty) { getAllOnesValue(VTy->getElementType())); } -/// getAggregateElement - For aggregates (struct/array/vector) return the -/// constant that corresponds to the specified element if possible, or null if -/// not. This can return null if the element index is a ConstantExpr, or if -/// 'this' is a constant expr. Constant *Constant::getAggregateElement(unsigned Elt) const { - if (const ConstantStruct *CS = dyn_cast<ConstantStruct>(this)) - return Elt < CS->getNumOperands() ? CS->getOperand(Elt) : nullptr; - - if (const ConstantArray *CA = dyn_cast<ConstantArray>(this)) - return Elt < CA->getNumOperands() ? CA->getOperand(Elt) : nullptr; - - if (const ConstantVector *CV = dyn_cast<ConstantVector>(this)) - return Elt < CV->getNumOperands() ? CV->getOperand(Elt) : nullptr; + if (const ConstantAggregate *CC = dyn_cast<ConstantAggregate>(this)) + return Elt < CC->getNumOperands() ? CC->getOperand(Elt) : nullptr; if (const ConstantAggregateZero *CAZ = dyn_cast<ConstantAggregateZero>(this)) return Elt < CAZ->getNumElements() ? CAZ->getElementValue(Elt) : nullptr; @@ -369,8 +358,6 @@ static bool canTrapImpl(const Constant *C, } } -/// canTrap - Return true if evaluation of this constant could trap. This is -/// true for things like constant expressions that could divide by zero. bool Constant::canTrap() const { SmallPtrSet<const ConstantExpr *, 4> NonTrappingOps; return canTrapImpl(this, NonTrappingOps); @@ -401,7 +388,6 @@ ConstHasGlobalValuePredicate(const Constant *C, return false; } -/// Return true if the value can vary between threads. bool Constant::isThreadDependent() const { auto DLLImportPredicate = [](const GlobalValue *GV) { return GV->isThreadLocal(); @@ -416,8 +402,6 @@ bool Constant::isDLLImportDependent() const { return ConstHasGlobalValuePredicate(this, DLLImportPredicate); } -/// Return true if the constant has users other than constant exprs and other -/// dangling things. bool Constant::isConstantUsed() const { for (const User *U : users()) { const Constant *UC = dyn_cast<Constant>(U); @@ -461,9 +445,8 @@ bool Constant::needsRelocation() const { return Result; } -/// removeDeadUsersOfConstant - If the specified constantexpr is dead, remove -/// it. This involves recursively eliminating any dead users of the -/// constantexpr. +/// If the specified constantexpr is dead, remove it. This involves recursively +/// eliminating any dead users of the constantexpr. static bool removeDeadUsersOfConstant(const Constant *C) { if (isa<GlobalValue>(C)) return false; // Cannot remove this @@ -479,10 +462,6 @@ static bool removeDeadUsersOfConstant(const Constant *C) { } -/// removeDeadConstantUsers - If there are any dead constant users dangling -/// off of this constant, remove them. This method is useful for clients -/// that want to check to see if a global is unused, but don't want to deal -/// with potentially dead constants hanging off of the globals. void Constant::removeDeadConstantUsers() const { Value::const_user_iterator I = user_begin(), E = user_end(); Value::const_user_iterator LastNonDeadUser = E; @@ -521,8 +500,8 @@ void Constant::removeDeadConstantUsers() const { void ConstantInt::anchor() { } -ConstantInt::ConstantInt(IntegerType *Ty, const APInt& V) - : Constant(Ty, ConstantIntVal, nullptr, 0), Val(V) { +ConstantInt::ConstantInt(IntegerType *Ty, const APInt &V) + : ConstantData(Ty, ConstantIntVal), Val(V) { assert(V.getBitWidth() == Ty->getBitWidth() && "Invalid constant for type"); } @@ -588,8 +567,7 @@ Constant *ConstantInt::get(Type *Ty, uint64_t V, bool isSigned) { return C; } -ConstantInt *ConstantInt::get(IntegerType *Ty, uint64_t V, - bool isSigned) { +ConstantInt *ConstantInt::get(IntegerType *Ty, uint64_t V, bool isSigned) { return get(Ty->getContext(), APInt(Ty->getBitWidth(), V, isSigned)); } @@ -613,8 +591,7 @@ Constant *ConstantInt::get(Type *Ty, const APInt& V) { return C; } -ConstantInt *ConstantInt::get(IntegerType* Ty, StringRef Str, - uint8_t radix) { +ConstantInt *ConstantInt::get(IntegerType* Ty, StringRef Str, uint8_t radix) { return get(Ty->getContext(), APInt(Ty->getBitWidth(), Str, radix)); } @@ -645,9 +622,6 @@ static const fltSemantics *TypeToFloatSemantics(Type *Ty) { void ConstantFP::anchor() { } -/// get() - This returns a constant fp for the specified value in the -/// specified type. This should only be used for simple constant values like -/// 2.0/1.0 etc, that are known-valid both as double and as the target format. Constant *ConstantFP::get(Type *Ty, double V) { LLVMContext &Context = Ty->getContext(); @@ -748,8 +722,8 @@ Constant *ConstantFP::getInfinity(Type *Ty, bool Negative) { return C; } -ConstantFP::ConstantFP(Type *Ty, const APFloat& V) - : Constant(Ty, ConstantFPVal, nullptr, 0), Val(V) { +ConstantFP::ConstantFP(Type *Ty, const APFloat &V) + : ConstantData(Ty, ConstantFPVal), Val(V) { assert(&V.getSemantics() == TypeToFloatSemantics(Ty) && "FP type Mismatch"); } @@ -767,28 +741,20 @@ void ConstantFP::destroyConstantImpl() { // ConstantAggregateZero Implementation //===----------------------------------------------------------------------===// -/// getSequentialElement - If this CAZ has array or vector type, return a zero -/// with the right element type. Constant *ConstantAggregateZero::getSequentialElement() const { return Constant::getNullValue(getType()->getSequentialElementType()); } -/// getStructElement - If this CAZ has struct type, return a zero with the -/// right element type for the specified element. Constant *ConstantAggregateZero::getStructElement(unsigned Elt) const { return Constant::getNullValue(getType()->getStructElementType(Elt)); } -/// getElementValue - Return a zero of the right value for the specified GEP -/// index if we can, otherwise return null (e.g. if C is a ConstantExpr). Constant *ConstantAggregateZero::getElementValue(Constant *C) const { if (isa<SequentialType>(getType())) return getSequentialElement(); return getStructElement(cast<ConstantInt>(C)->getZExtValue()); } -/// getElementValue - Return a zero of the right value for the specified GEP -/// index. Constant *ConstantAggregateZero::getElementValue(unsigned Idx) const { if (isa<SequentialType>(getType())) return getSequentialElement(); @@ -808,28 +774,20 @@ unsigned ConstantAggregateZero::getNumElements() const { // UndefValue Implementation //===----------------------------------------------------------------------===// -/// getSequentialElement - If this undef has array or vector type, return an -/// undef with the right element type. UndefValue *UndefValue::getSequentialElement() const { return UndefValue::get(getType()->getSequentialElementType()); } -/// getStructElement - If this undef has struct type, return a zero with the -/// right element type for the specified element. UndefValue *UndefValue::getStructElement(unsigned Elt) const { return UndefValue::get(getType()->getStructElementType(Elt)); } -/// getElementValue - Return an undef of the right value for the specified GEP -/// index if we can, otherwise return null (e.g. if C is a ConstantExpr). UndefValue *UndefValue::getElementValue(Constant *C) const { if (isa<SequentialType>(getType())) return getSequentialElement(); return getStructElement(cast<ConstantInt>(C)->getZExtValue()); } -/// getElementValue - Return an undef of the right value for the specified GEP -/// index. UndefValue *UndefValue::getElementValue(unsigned Idx) const { if (isa<SequentialType>(getType())) return getSequentialElement(); @@ -910,16 +868,25 @@ static Constant *getSequenceIfElementsMatch(Constant *C, return nullptr; } +ConstantAggregate::ConstantAggregate(CompositeType *T, ValueTy VT, + ArrayRef<Constant *> V) + : Constant(T, VT, OperandTraits<ConstantAggregate>::op_end(this) - V.size(), + V.size()) { + std::copy(V.begin(), V.end(), op_begin()); + + // Check that types match, unless this is an opaque struct. + if (auto *ST = dyn_cast<StructType>(T)) + if (ST->isOpaque()) + return; + for (unsigned I = 0, E = V.size(); I != E; ++I) + assert(V[I]->getType() == T->getTypeAtIndex(I) && + "Initializer for composite element doesn't match!"); +} + ConstantArray::ConstantArray(ArrayType *T, ArrayRef<Constant *> V) - : Constant(T, ConstantArrayVal, - OperandTraits<ConstantArray>::op_end(this) - V.size(), - V.size()) { + : ConstantAggregate(T, ConstantArrayVal, V) { assert(V.size() == T->getNumElements() && - "Invalid initializer vector for constant array"); - for (unsigned i = 0, e = V.size(); i != e; ++i) - assert(V[i]->getType() == T->getElementType() && - "Initializer for array element doesn't match array element type!"); - std::copy(V.begin(), V.end(), op_begin()); + "Invalid initializer for constant array"); } Constant *ConstantArray::get(ArrayType *Ty, ArrayRef<Constant*> V) { @@ -957,8 +924,6 @@ Constant *ConstantArray::getImpl(ArrayType *Ty, ArrayRef<Constant*> V) { return nullptr; } -/// getTypeForElements - Return an anonymous struct type to use for a constant -/// with the specified set of elements. The list must not be empty. StructType *ConstantStruct::getTypeForElements(LLVMContext &Context, ArrayRef<Constant*> V, bool Packed) { @@ -978,17 +943,10 @@ StructType *ConstantStruct::getTypeForElements(ArrayRef<Constant*> V, return getTypeForElements(V[0]->getContext(), V, Packed); } - ConstantStruct::ConstantStruct(StructType *T, ArrayRef<Constant *> V) - : Constant(T, ConstantStructVal, - OperandTraits<ConstantStruct>::op_end(this) - V.size(), - V.size()) { - assert(V.size() == T->getNumElements() && - "Invalid initializer vector for constant structure"); - for (unsigned i = 0, e = V.size(); i != e; ++i) - assert((T->isOpaque() || V[i]->getType() == T->getElementType(i)) && - "Initializer for struct element doesn't match struct element type!"); - std::copy(V.begin(), V.end(), op_begin()); + : ConstantAggregate(T, ConstantStructVal, V) { + assert((T->isOpaque() || V.size() == T->getNumElements()) && + "Invalid initializer for constant struct"); } // ConstantStruct accessors. @@ -1031,13 +989,9 @@ Constant *ConstantStruct::get(StructType *T, ...) { } ConstantVector::ConstantVector(VectorType *T, ArrayRef<Constant *> V) - : Constant(T, ConstantVectorVal, - OperandTraits<ConstantVector>::op_end(this) - V.size(), - V.size()) { - for (size_t i = 0, e = V.size(); i != e; i++) - assert(V[i]->getType() == T->getElementType() && - "Initializer for vector element doesn't match vector element type!"); - std::copy(V.begin(), V.end(), op_begin()); + : ConstantAggregate(T, ConstantVectorVal, V) { + assert(V.size() == T->getNumElements() && + "Invalid initializer for constant vector"); } // ConstantVector accessors. @@ -1157,8 +1111,6 @@ unsigned ConstantExpr::getPredicate() const { return cast<CompareConstantExpr>(this)->predicate; } -/// getWithOperandReplaced - Return a constant expression identical to this -/// one, but with the specified operand set to the specified value. Constant * ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const { assert(Op->getType() == getOperand(OpNo)->getType() && @@ -1173,9 +1125,6 @@ ConstantExpr::getWithOperandReplaced(unsigned OpNo, Constant *Op) const { return getWithOperands(NewOps); } -/// getWithOperands - This returns the current constant expression with the -/// operands replaced with the specified values. The specified array must -/// have the same number of operands as our current one. Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty, bool OnlyIfReduced, Type *SrcTy) const { assert(Ops.size() == getNumOperands() && "Operand count mismatch!"); @@ -1320,14 +1269,12 @@ ConstantAggregateZero *ConstantAggregateZero::get(Type *Ty) { return Entry; } -/// destroyConstant - Remove the constant from the constant table. -/// +/// Remove the constant from the constant table. void ConstantAggregateZero::destroyConstantImpl() { getContext().pImpl->CAZConstants.erase(getType()); } -/// destroyConstant - Remove the constant from the constant table... -/// +/// Remove the constant from the constant table. void ConstantArray::destroyConstantImpl() { getType()->getContext().pImpl->ArrayConstants.remove(this); } @@ -1336,20 +1283,16 @@ void ConstantArray::destroyConstantImpl() { //---- ConstantStruct::get() implementation... // -// destroyConstant - Remove the constant from the constant table... -// +/// Remove the constant from the constant table. void ConstantStruct::destroyConstantImpl() { getType()->getContext().pImpl->StructConstants.remove(this); } -// destroyConstant - Remove the constant from the constant table... -// +/// Remove the constant from the constant table. void ConstantVector::destroyConstantImpl() { getType()->getContext().pImpl->VectorConstants.remove(this); } -/// getSplatValue - If this is a splat vector constant, meaning that all of -/// the elements have the same value, return that value. Otherwise return 0. Constant *Constant::getSplatValue() const { assert(this->getType()->isVectorTy() && "Only valid for vectors!"); if (isa<ConstantAggregateZero>(this)) @@ -1361,8 +1304,6 @@ Constant *Constant::getSplatValue() const { return nullptr; } -/// getSplatValue - If this is a splat constant, where all of the -/// elements have the same value, return that value. Otherwise return null. Constant *ConstantVector::getSplatValue() const { // Check out first element. Constant *Elt = getOperand(0); @@ -1373,8 +1314,6 @@ Constant *ConstantVector::getSplatValue() const { return Elt; } -/// If C is a constant integer then return its value, otherwise C must be a -/// vector of constant integers, all equal, and the common value is returned. const APInt &Constant::getUniqueInteger() const { if (const ConstantInt *CI = dyn_cast<ConstantInt>(this)) return CI->getValue(); @@ -1395,16 +1334,11 @@ ConstantPointerNull *ConstantPointerNull::get(PointerType *Ty) { return Entry; } -// destroyConstant - Remove the constant from the constant table... -// +/// Remove the constant from the constant table. void ConstantPointerNull::destroyConstantImpl() { getContext().pImpl->CPNConstants.erase(getType()); } - -//---- UndefValue::get() implementation. -// - UndefValue *UndefValue::get(Type *Ty) { UndefValue *&Entry = Ty->getContext().pImpl->UVConstants[Ty]; if (!Entry) @@ -1413,16 +1347,12 @@ UndefValue *UndefValue::get(Type *Ty) { return Entry; } -// destroyConstant - Remove the constant from the constant table. -// +/// Remove the constant from the constant table. void UndefValue::destroyConstantImpl() { // Free the constant and any dangling references to it. getContext().pImpl->UVConstants.erase(getType()); } -//---- BlockAddress::get() implementation. -// - BlockAddress *BlockAddress::get(BasicBlock *BB) { assert(BB->getParent() && "Block must have a parent"); return get(BB->getParent(), BB); @@ -1458,24 +1388,25 @@ BlockAddress *BlockAddress::lookup(const BasicBlock *BB) { return BA; } -// destroyConstant - Remove the constant from the constant table. -// +/// Remove the constant from the constant table. void BlockAddress::destroyConstantImpl() { getFunction()->getType()->getContext().pImpl ->BlockAddresses.erase(std::make_pair(getFunction(), getBasicBlock())); getBasicBlock()->AdjustBlockAddressRefCount(-1); } -Value *BlockAddress::handleOperandChangeImpl(Value *From, Value *To, Use *U) { +Value *BlockAddress::handleOperandChangeImpl(Value *From, Value *To) { // This could be replacing either the Basic Block or the Function. In either // case, we have to remove the map entry. Function *NewF = getFunction(); BasicBlock *NewBB = getBasicBlock(); - if (U == &Op<0>()) + if (From == NewF) NewF = cast<Function>(To->stripPointerCasts()); - else + else { + assert(From == NewBB && "From does not match any operand"); NewBB = cast<BasicBlock>(To); + } // See if the 'new' entry already exists, if not, just update this in place // and return early. @@ -1606,8 +1537,7 @@ Constant *ConstantExpr::getPointerBitCastOrAddrSpaceCast(Constant *S, return getBitCast(S, Ty); } -Constant *ConstantExpr::getIntegerCast(Constant *C, Type *Ty, - bool isSigned) { +Constant *ConstantExpr::getIntegerCast(Constant *C, Type *Ty, bool isSigned) { assert(C->getType()->isIntOrIntVectorTy() && Ty->isIntOrIntVectorTy() && "Invalid cast"); unsigned SrcBits = C->getType()->getScalarSizeInBits(); @@ -1979,8 +1909,16 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, assert(DestTy && "GEP indices invalid!"); unsigned AS = C->getType()->getPointerAddressSpace(); Type *ReqTy = DestTy->getPointerTo(AS); - if (VectorType *VecTy = dyn_cast<VectorType>(C->getType())) - ReqTy = VectorType::get(ReqTy, VecTy->getNumElements()); + + unsigned NumVecElts = 0; + if (C->getType()->isVectorTy()) + NumVecElts = C->getType()->getVectorNumElements(); + else for (auto Idx : Idxs) + if (Idx->getType()->isVectorTy()) + NumVecElts = Idx->getType()->getVectorNumElements(); + + if (NumVecElts) + ReqTy = VectorType::get(ReqTy, NumVecElts); if (OnlyIfReducedTy == ReqTy) return nullptr; @@ -1990,13 +1928,14 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C, ArgVec.reserve(1 + Idxs.size()); ArgVec.push_back(C); for (unsigned i = 0, e = Idxs.size(); i != e; ++i) { - assert(Idxs[i]->getType()->isVectorTy() == ReqTy->isVectorTy() && - "getelementptr index type missmatch"); assert((!Idxs[i]->getType()->isVectorTy() || - ReqTy->getVectorNumElements() == - Idxs[i]->getType()->getVectorNumElements()) && + Idxs[i]->getType()->getVectorNumElements() == NumVecElts) && "getelementptr index type missmatch"); - ArgVec.push_back(cast<Constant>(Idxs[i])); + + Constant *Idx = cast<Constant>(Idxs[i]); + if (NumVecElts && !Idxs[i]->getType()->isVectorTy()) + Idx = ConstantVector::getSplat(NumVecElts, Idx); + ArgVec.push_back(Idx); } const ConstantExprKeyType Key(Instruction::GetElementPtr, ArgVec, 0, InBounds ? GEPOperator::IsInBounds : 0, None, @@ -2278,9 +2217,6 @@ Constant *ConstantExpr::getAShr(Constant *C1, Constant *C2, bool isExact) { isExact ? PossiblyExactOperator::IsExact : 0); } -/// getBinOpIdentity - Return the identity for the given binary operation, -/// i.e. a constant C such that X op C = X and C op X = X for every X. It -/// returns null if the operator doesn't have an identity. Constant *ConstantExpr::getBinOpIdentity(unsigned Opcode, Type *Ty) { switch (Opcode) { default: @@ -2300,10 +2236,6 @@ Constant *ConstantExpr::getBinOpIdentity(unsigned Opcode, Type *Ty) { } } -/// getBinOpAbsorber - Return the absorbing element for the given binary -/// operation, i.e. a constant C such that X op C = C and C op X = C for -/// every X. For example, this returns zero for integer multiplication. -/// It returns null if the operator doesn't have an absorbing element. Constant *ConstantExpr::getBinOpAbsorber(unsigned Opcode, Type *Ty) { switch (Opcode) { default: @@ -2319,8 +2251,7 @@ Constant *ConstantExpr::getBinOpAbsorber(unsigned Opcode, Type *Ty) { } } -// destroyConstant - Remove the constant from the constant table... -// +/// Remove the constant from the constant table. void ConstantExpr::destroyConstantImpl() { getType()->getContext().pImpl->ExprConstants.remove(this); } @@ -2335,7 +2266,8 @@ GetElementPtrConstantExpr::GetElementPtrConstantExpr( OperandTraits<GetElementPtrConstantExpr>::op_end(this) - (IdxList.size() + 1), IdxList.size() + 1), - SrcElementTy(SrcElementTy) { + SrcElementTy(SrcElementTy), + ResElementTy(GetElementPtrInst::getIndexedType(SrcElementTy, IdxList)) { Op<0>() = C; Use *OperandList = getOperandList(); for (unsigned i = 0, E = IdxList.size(); i != E; ++i) @@ -2346,13 +2278,16 @@ Type *GetElementPtrConstantExpr::getSourceElementType() const { return SrcElementTy; } +Type *GetElementPtrConstantExpr::getResultElementType() const { + return ResElementTy; +} + //===----------------------------------------------------------------------===// // ConstantData* implementations void ConstantDataArray::anchor() {} void ConstantDataVector::anchor() {} -/// getElementType - Return the element type of the array/vector. Type *ConstantDataSequential::getElementType() const { return getType()->getElementType(); } @@ -2361,10 +2296,6 @@ StringRef ConstantDataSequential::getRawDataValues() const { return StringRef(DataElements, getNumElements()*getElementByteSize()); } -/// isElementTypeCompatible - Return true if a ConstantDataSequential can be -/// formed with a vector or array of the specified element type. -/// ConstantDataArray only works with normal float and int types that are -/// stored densely in memory, not with things like i42 or x86_f80. bool ConstantDataSequential::isElementTypeCompatible(Type *Ty) { if (Ty->isHalfTy() || Ty->isFloatTy() || Ty->isDoubleTy()) return true; if (auto *IT = dyn_cast<IntegerType>(Ty)) { @@ -2380,7 +2311,6 @@ bool ConstantDataSequential::isElementTypeCompatible(Type *Ty) { return false; } -/// getNumElements - Return the number of elements in the array or vector. unsigned ConstantDataSequential::getNumElements() const { if (ArrayType *AT = dyn_cast<ArrayType>(getType())) return AT->getNumElements(); @@ -2388,27 +2318,26 @@ unsigned ConstantDataSequential::getNumElements() const { } -/// getElementByteSize - Return the size in bytes of the elements in the data. uint64_t ConstantDataSequential::getElementByteSize() const { return getElementType()->getPrimitiveSizeInBits()/8; } -/// getElementPointer - Return the start of the specified element. +/// Return the start of the specified element. const char *ConstantDataSequential::getElementPointer(unsigned Elt) const { assert(Elt < getNumElements() && "Invalid Elt"); return DataElements+Elt*getElementByteSize(); } -/// isAllZeros - return true if the array is empty or all zeros. +/// Return true if the array is empty or all zeros. static bool isAllZeros(StringRef Arr) { - for (StringRef::iterator I = Arr.begin(), E = Arr.end(); I != E; ++I) - if (*I != 0) + for (char I : Arr) + if (I != 0) return false; return true; } -/// getImpl - This is the underlying implementation of all of the +/// This is the underlying implementation of all of the /// ConstantDataSequential::get methods. They all thunk down to here, providing /// the correct element type. We take the bytes in as a StringRef because /// we *want* an underlying "char*" to avoid TBAA type punning violations. @@ -2539,11 +2468,6 @@ Constant *ConstantDataArray::getFP(LLVMContext &Context, return getImpl(StringRef(const_cast<char *>(Data), Elts.size() * 8), Ty); } -/// getString - This method constructs a CDS and initializes it with a text -/// string. The default behavior (AddNull==true) causes a null terminator to -/// be placed at the end of the array (increasing the length of the string by -/// one more than the StringRef would normally indicate. Pass AddNull=false -/// to disable this behavior. Constant *ConstantDataArray::getString(LLVMContext &Context, StringRef Str, bool AddNull) { if (!AddNull) { @@ -2658,8 +2582,6 @@ Constant *ConstantDataVector::getSplat(unsigned NumElts, Constant *V) { } -/// getElementAsInteger - If this is a sequential container of integers (of -/// any size), return the specified element in the low bits of a uint64_t. uint64_t ConstantDataSequential::getElementAsInteger(unsigned Elt) const { assert(isa<IntegerType>(getElementType()) && "Accessor can only be used when element is an integer"); @@ -2680,8 +2602,6 @@ uint64_t ConstantDataSequential::getElementAsInteger(unsigned Elt) const { } } -/// getElementAsAPFloat - If this is a sequential container of floating point -/// type, return the specified element as an APFloat. APFloat ConstantDataSequential::getElementAsAPFloat(unsigned Elt) const { const char *EltPtr = getElementPointer(Elt); @@ -2703,8 +2623,6 @@ APFloat ConstantDataSequential::getElementAsAPFloat(unsigned Elt) const { } } -/// getElementAsFloat - If this is an sequential container of floats, return -/// the specified element as a float. float ConstantDataSequential::getElementAsFloat(unsigned Elt) const { assert(getElementType()->isFloatTy() && "Accessor can only be used when element is a 'float'"); @@ -2712,8 +2630,6 @@ float ConstantDataSequential::getElementAsFloat(unsigned Elt) const { return *const_cast<float *>(EltPtr); } -/// getElementAsDouble - If this is an sequential container of doubles, return -/// the specified element as a float. double ConstantDataSequential::getElementAsDouble(unsigned Elt) const { assert(getElementType()->isDoubleTy() && "Accessor can only be used when element is a 'float'"); @@ -2722,9 +2638,6 @@ double ConstantDataSequential::getElementAsDouble(unsigned Elt) const { return *const_cast<double *>(EltPtr); } -/// getElementAsConstant - Return a Constant for a specified index's element. -/// Note that this has to compute a new constant to return, so it isn't as -/// efficient as getElementAsInteger/Float/Double. Constant *ConstantDataSequential::getElementAsConstant(unsigned Elt) const { if (getElementType()->isHalfTy() || getElementType()->isFloatTy() || getElementType()->isDoubleTy()) @@ -2733,13 +2646,10 @@ Constant *ConstantDataSequential::getElementAsConstant(unsigned Elt) const { return ConstantInt::get(getElementType(), getElementAsInteger(Elt)); } -/// isString - This method returns true if this is an array of i8. bool ConstantDataSequential::isString() const { return isa<ArrayType>(getType()) && getElementType()->isIntegerTy(8); } -/// isCString - This method returns true if the array "isString", ends with a -/// nul byte, and does not contains any other nul bytes. bool ConstantDataSequential::isCString() const { if (!isString()) return false; @@ -2753,8 +2663,6 @@ bool ConstantDataSequential::isCString() const { return Str.drop_back().find(0) == StringRef::npos; } -/// getSplatValue - If this is a splat constant, meaning that all of the -/// elements have the same value, return that value. Otherwise return nullptr. Constant *ConstantDataVector::getSplatValue() const { const char *Base = getRawDataValues().data(); @@ -2782,14 +2690,14 @@ Constant *ConstantDataVector::getSplatValue() const { /// work, but would be really slow because it would have to unique each updated /// array instance. /// -void Constant::handleOperandChange(Value *From, Value *To, Use *U) { +void Constant::handleOperandChange(Value *From, Value *To) { Value *Replacement = nullptr; switch (getValueID()) { default: llvm_unreachable("Not a constant!"); #define HANDLE_CONSTANT(Name) \ case Value::Name##Val: \ - Replacement = cast<Name>(this)->handleOperandChangeImpl(From, To, U); \ + Replacement = cast<Name>(this)->handleOperandChangeImpl(From, To); \ break; #include "llvm/IR/Value.def" } @@ -2809,39 +2717,7 @@ void Constant::handleOperandChange(Value *From, Value *To, Use *U) { destroyConstant(); } -Value *ConstantInt::handleOperandChangeImpl(Value *From, Value *To, Use *U) { - llvm_unreachable("Unsupported class for handleOperandChange()!"); -} - -Value *ConstantFP::handleOperandChangeImpl(Value *From, Value *To, Use *U) { - llvm_unreachable("Unsupported class for handleOperandChange()!"); -} - -Value *ConstantTokenNone::handleOperandChangeImpl(Value *From, Value *To, - Use *U) { - llvm_unreachable("Unsupported class for handleOperandChange()!"); -} - -Value *UndefValue::handleOperandChangeImpl(Value *From, Value *To, Use *U) { - llvm_unreachable("Unsupported class for handleOperandChange()!"); -} - -Value *ConstantPointerNull::handleOperandChangeImpl(Value *From, Value *To, - Use *U) { - llvm_unreachable("Unsupported class for handleOperandChange()!"); -} - -Value *ConstantAggregateZero::handleOperandChangeImpl(Value *From, Value *To, - Use *U) { - llvm_unreachable("Unsupported class for handleOperandChange()!"); -} - -Value *ConstantDataSequential::handleOperandChangeImpl(Value *From, Value *To, - Use *U) { - llvm_unreachable("Unsupported class for handleOperandChange()!"); -} - -Value *ConstantArray::handleOperandChangeImpl(Value *From, Value *To, Use *U) { +Value *ConstantArray::handleOperandChangeImpl(Value *From, Value *To) { assert(isa<Constant>(To) && "Cannot make Constant refer to non-constant!"); Constant *ToC = cast<Constant>(To); @@ -2855,9 +2731,11 @@ Value *ConstantArray::handleOperandChangeImpl(Value *From, Value *To, Use *U) { // Keep track of whether all the values in the array are "ToC". bool AllSame = true; Use *OperandList = getOperandList(); + unsigned OperandNo = 0; for (Use *O = OperandList, *E = OperandList+getNumOperands(); O != E; ++O) { Constant *Val = cast<Constant>(O->get()); if (Val == From) { + OperandNo = (O - OperandList); Val = ToC; ++NumUpdated; } @@ -2877,65 +2755,57 @@ Value *ConstantArray::handleOperandChangeImpl(Value *From, Value *To, Use *U) { // Update to the new value. return getContext().pImpl->ArrayConstants.replaceOperandsInPlace( - Values, this, From, ToC, NumUpdated, U - OperandList); + Values, this, From, ToC, NumUpdated, OperandNo); } -Value *ConstantStruct::handleOperandChangeImpl(Value *From, Value *To, Use *U) { +Value *ConstantStruct::handleOperandChangeImpl(Value *From, Value *To) { assert(isa<Constant>(To) && "Cannot make Constant refer to non-constant!"); Constant *ToC = cast<Constant>(To); Use *OperandList = getOperandList(); - unsigned OperandToUpdate = U-OperandList; - assert(getOperand(OperandToUpdate) == From && "ReplaceAllUsesWith broken!"); SmallVector<Constant*, 8> Values; Values.reserve(getNumOperands()); // Build replacement struct. // Fill values with the modified operands of the constant struct. Also, // compute whether this turns into an all-zeros struct. - bool isAllZeros = false; - bool isAllUndef = false; - if (ToC->isNullValue()) { - isAllZeros = true; - for (Use *O = OperandList, *E = OperandList+getNumOperands(); O != E; ++O) { - Constant *Val = cast<Constant>(O->get()); - Values.push_back(Val); - if (isAllZeros) isAllZeros = Val->isNullValue(); - } - } else if (isa<UndefValue>(ToC)) { - isAllUndef = true; - for (Use *O = OperandList, *E = OperandList+getNumOperands(); O != E; ++O) { - Constant *Val = cast<Constant>(O->get()); - Values.push_back(Val); - if (isAllUndef) isAllUndef = isa<UndefValue>(Val); + unsigned NumUpdated = 0; + bool AllSame = true; + unsigned OperandNo = 0; + for (Use *O = OperandList, *E = OperandList + getNumOperands(); O != E; ++O) { + Constant *Val = cast<Constant>(O->get()); + if (Val == From) { + OperandNo = (O - OperandList); + Val = ToC; + ++NumUpdated; } - } else { - for (Use *O = OperandList, *E = OperandList + getNumOperands(); O != E; ++O) - Values.push_back(cast<Constant>(O->get())); + Values.push_back(Val); + AllSame &= Val == ToC; } - Values[OperandToUpdate] = ToC; - if (isAllZeros) + if (AllSame && ToC->isNullValue()) return ConstantAggregateZero::get(getType()); - if (isAllUndef) + if (AllSame && isa<UndefValue>(ToC)) return UndefValue::get(getType()); // Update to the new value. return getContext().pImpl->StructConstants.replaceOperandsInPlace( - Values, this, From, ToC); + Values, this, From, ToC, NumUpdated, OperandNo); } -Value *ConstantVector::handleOperandChangeImpl(Value *From, Value *To, Use *U) { +Value *ConstantVector::handleOperandChangeImpl(Value *From, Value *To) { assert(isa<Constant>(To) && "Cannot make Constant refer to non-constant!"); Constant *ToC = cast<Constant>(To); SmallVector<Constant*, 8> Values; Values.reserve(getNumOperands()); // Build replacement array... unsigned NumUpdated = 0; + unsigned OperandNo = 0; for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { Constant *Val = getOperand(i); if (Val == From) { + OperandNo = i; ++NumUpdated; Val = ToC; } @@ -2946,20 +2816,21 @@ Value *ConstantVector::handleOperandChangeImpl(Value *From, Value *To, Use *U) { return C; // Update to the new value. - Use *OperandList = getOperandList(); return getContext().pImpl->VectorConstants.replaceOperandsInPlace( - Values, this, From, ToC, NumUpdated, U - OperandList); + Values, this, From, ToC, NumUpdated, OperandNo); } -Value *ConstantExpr::handleOperandChangeImpl(Value *From, Value *ToV, Use *U) { +Value *ConstantExpr::handleOperandChangeImpl(Value *From, Value *ToV) { assert(isa<Constant>(ToV) && "Cannot make Constant refer to non-constant!"); Constant *To = cast<Constant>(ToV); SmallVector<Constant*, 8> NewOps; unsigned NumUpdated = 0; + unsigned OperandNo = 0; for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { Constant *Op = getOperand(i); if (Op == From) { + OperandNo = i; ++NumUpdated; Op = To; } @@ -2971,9 +2842,8 @@ Value *ConstantExpr::handleOperandChangeImpl(Value *From, Value *ToV, Use *U) { return C; // Update to the new value. - Use *OperandList = getOperandList(); return getContext().pImpl->ExprConstants.replaceOperandsInPlace( - NewOps, this, From, To, NumUpdated, U - OperandList); + NewOps, this, From, To, NumUpdated, OperandNo); } Instruction *ConstantExpr::getAsInstruction() { diff --git a/lib/IR/ConstantsContext.h b/lib/IR/ConstantsContext.h index 13fcbd2ece10a..7db87edff010a 100644 --- a/lib/IR/ConstantsContext.h +++ b/lib/IR/ConstantsContext.h @@ -15,7 +15,7 @@ #ifndef LLVM_LIB_IR_CONSTANTSCONTEXT_H #define LLVM_LIB_IR_CONSTANTSCONTEXT_H -#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/Hashing.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Instructions.h" @@ -23,8 +23,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include <map> -#include <tuple> #define DEBUG_TYPE "ir" @@ -225,19 +223,12 @@ public: /// used behind the scenes to implement getelementpr constant exprs. class GetElementPtrConstantExpr : public ConstantExpr { Type *SrcElementTy; + Type *ResElementTy; void anchor() override; GetElementPtrConstantExpr(Type *SrcElementTy, Constant *C, ArrayRef<Constant *> IdxList, Type *DestTy); public: - static GetElementPtrConstantExpr *Create(Constant *C, - ArrayRef<Constant*> IdxList, - Type *DestTy, - unsigned Flags) { - return Create( - cast<PointerType>(C->getType()->getScalarType())->getElementType(), C, - IdxList, DestTy, Flags); - } static GetElementPtrConstantExpr *Create(Type *SrcElementTy, Constant *C, ArrayRef<Constant *> IdxList, Type *DestTy, unsigned Flags) { @@ -247,6 +238,7 @@ public: return Result; } Type *getSourceElementType() const; + Type *getResultElementType() const; /// Transparently provide more efficient getOperand methods. DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); @@ -552,6 +544,9 @@ public: typedef typename ConstantInfo<ConstantClass>::TypeClass TypeClass; typedef std::pair<TypeClass *, ValType> LookupKey; + /// Key and hash together, so that we compute the hash only once and reuse it. + typedef std::pair<unsigned, LookupKey> LookupKeyHashed; + private: struct MapInfo { typedef DenseMapInfo<ConstantClass *> ConstantClassInfo; @@ -562,7 +557,7 @@ private: return ConstantClassInfo::getTombstoneKey(); } static unsigned getHashValue(const ConstantClass *CP) { - SmallVector<Constant *, 8> Storage; + SmallVector<Constant *, 32> Storage; return getHashValue(LookupKey(CP->getType(), ValType(CP, Storage))); } static bool isEqual(const ConstantClass *LHS, const ConstantClass *RHS) { @@ -571,6 +566,9 @@ private: static unsigned getHashValue(const LookupKey &Val) { return hash_combine(Val.first, Val.second.getHash()); } + static unsigned getHashValue(const LookupKeyHashed &Val) { + return Val.first; + } static bool isEqual(const LookupKey &LHS, const ConstantClass *RHS) { if (RHS == getEmptyKey() || RHS == getTombstoneKey()) return false; @@ -578,30 +576,31 @@ private: return false; return LHS.second == RHS; } + static bool isEqual(const LookupKeyHashed &LHS, const ConstantClass *RHS) { + return isEqual(LHS.second, RHS); + } }; public: - typedef DenseMap<ConstantClass *, char, MapInfo> MapTy; + typedef DenseSet<ConstantClass *, MapInfo> MapTy; private: MapTy Map; public: - typename MapTy::iterator map_begin() { return Map.begin(); } - typename MapTy::iterator map_end() { return Map.end(); } + typename MapTy::iterator begin() { return Map.begin(); } + typename MapTy::iterator end() { return Map.end(); } void freeConstants() { for (auto &I : Map) - // Asserts that use_empty(). - delete I.first; + delete I; // Asserts that use_empty(). } - private: - ConstantClass *create(TypeClass *Ty, ValType V) { + ConstantClass *create(TypeClass *Ty, ValType V, LookupKeyHashed &HashKey) { ConstantClass *Result = V.create(Ty); assert(Result->getType() == Ty && "Type specified is not correct!"); - insert(Result); + Map.insert_as(Result, HashKey); return Result; } @@ -609,32 +608,27 @@ private: public: /// Return the specified constant from the map, creating it if necessary. ConstantClass *getOrCreate(TypeClass *Ty, ValType V) { - LookupKey Lookup(Ty, V); + LookupKey Key(Ty, V); + /// Hash once, and reuse it for the lookup and the insertion if needed. + LookupKeyHashed Lookup(MapInfo::getHashValue(Key), Key); + ConstantClass *Result = nullptr; - auto I = find(Lookup); + auto I = Map.find_as(Lookup); if (I == Map.end()) - Result = create(Ty, V); + Result = create(Ty, V, Lookup); else - Result = I->first; + Result = *I; assert(Result && "Unexpected nullptr"); return Result; } - /// Find the constant by lookup key. - typename MapTy::iterator find(LookupKey Lookup) { - return Map.find_as(Lookup); - } - - /// Insert the constant into its proper slot. - void insert(ConstantClass *CP) { Map[CP] = '\0'; } - /// Remove this constant from the map void remove(ConstantClass *CP) { typename MapTy::iterator I = Map.find(CP); assert(I != Map.end() && "Constant not found in constant table!"); - assert(I->first == CP && "Didn't find correct element?"); + assert(*I == CP && "Didn't find correct element?"); Map.erase(I); } @@ -642,10 +636,13 @@ public: ConstantClass *CP, Value *From, Constant *To, unsigned NumUpdated = 0, unsigned OperandNo = ~0u) { - LookupKey Lookup(CP->getType(), ValType(Operands, CP)); - auto I = find(Lookup); + LookupKey Key(CP->getType(), ValType(Operands, CP)); + /// Hash once, and reuse it for the lookup and the insertion if needed. + LookupKeyHashed Lookup(MapInfo::getHashValue(Key), Key); + + auto I = Map.find_as(Lookup); if (I != Map.end()) - return I->first; + return *I; // Update to the new value. Optimize for the case when we have a single // operand that we're changing, but handle bulk updates efficiently. @@ -659,7 +656,7 @@ public: if (CP->getOperand(I) == From) CP->setOperand(I, To); } - insert(CP); + Map.insert_as(CP, Lookup); return nullptr; } diff --git a/lib/IR/Core.cpp b/lib/IR/Core.cpp index 591dafa22a4b0..a55361489adac 100644 --- a/lib/IR/Core.cpp +++ b/lib/IR/Core.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "llvm-c/Core.h" +#include "llvm/ADT/StringSwitch.h" #include "llvm/Bitcode/ReaderWriter.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/CallSite.h" @@ -73,22 +74,32 @@ void LLVMDisposeMessage(char *Message) { /*===-- Operations on contexts --------------------------------------------===*/ +static ManagedStatic<LLVMContext> GlobalContext; + LLVMContextRef LLVMContextCreate() { return wrap(new LLVMContext()); } -LLVMContextRef LLVMGetGlobalContext() { - return wrap(&getGlobalContext()); -} +LLVMContextRef LLVMGetGlobalContext() { return wrap(&*GlobalContext); } void LLVMContextSetDiagnosticHandler(LLVMContextRef C, LLVMDiagnosticHandler Handler, void *DiagnosticContext) { unwrap(C)->setDiagnosticHandler( - LLVM_EXTENSION reinterpret_cast<LLVMContext::DiagnosticHandlerTy>(Handler), + LLVM_EXTENSION reinterpret_cast<LLVMContext::DiagnosticHandlerTy>( + Handler), DiagnosticContext); } +LLVMDiagnosticHandler LLVMContextGetDiagnosticHandler(LLVMContextRef C) { + return LLVM_EXTENSION reinterpret_cast<LLVMDiagnosticHandler>( + unwrap(C)->getDiagnosticHandler()); +} + +void *LLVMContextGetDiagnosticContext(LLVMContextRef C) { + return unwrap(C)->getDiagnosticContext(); +} + void LLVMContextSetYieldCallback(LLVMContextRef C, LLVMYieldCallback Callback, void *OpaqueHandle) { auto YieldCallback = @@ -100,15 +111,72 @@ void LLVMContextDispose(LLVMContextRef C) { delete unwrap(C); } -unsigned LLVMGetMDKindIDInContext(LLVMContextRef C, const char* Name, +unsigned LLVMGetMDKindIDInContext(LLVMContextRef C, const char *Name, unsigned SLen) { return unwrap(C)->getMDKindID(StringRef(Name, SLen)); } -unsigned LLVMGetMDKindID(const char* Name, unsigned SLen) { +unsigned LLVMGetMDKindID(const char *Name, unsigned SLen) { return LLVMGetMDKindIDInContext(LLVMGetGlobalContext(), Name, SLen); } +#define GET_ATTR_KIND_FROM_NAME +#include "AttributesCompatFunc.inc" + +unsigned LLVMGetEnumAttributeKindForName(const char *Name, size_t SLen) { + return getAttrKindFromName(StringRef(Name, SLen)); +} + +unsigned LLVMGetLastEnumAttributeKind(void) { + return Attribute::AttrKind::EndAttrKinds; +} + +LLVMAttributeRef LLVMCreateEnumAttribute(LLVMContextRef C, unsigned KindID, + uint64_t Val) { + return wrap(Attribute::get(*unwrap(C), (Attribute::AttrKind)KindID, Val)); +} + +unsigned LLVMGetEnumAttributeKind(LLVMAttributeRef A) { + return unwrap(A).getKindAsEnum(); +} + +uint64_t LLVMGetEnumAttributeValue(LLVMAttributeRef A) { + auto Attr = unwrap(A); + if (Attr.isEnumAttribute()) + return 0; + return Attr.getValueAsInt(); +} + +LLVMAttributeRef LLVMCreateStringAttribute(LLVMContextRef C, + const char *K, unsigned KLength, + const char *V, unsigned VLength) { + return wrap(Attribute::get(*unwrap(C), StringRef(K, KLength), + StringRef(V, VLength))); +} + +const char *LLVMGetStringAttributeKind(LLVMAttributeRef A, + unsigned *Length) { + auto S = unwrap(A).getKindAsString(); + *Length = S.size(); + return S.data(); +} + +const char *LLVMGetStringAttributeValue(LLVMAttributeRef A, + unsigned *Length) { + auto S = unwrap(A).getValueAsString(); + *Length = S.size(); + return S.data(); +} + +LLVMBool LLVMIsEnumAttribute(LLVMAttributeRef A) { + auto Attr = unwrap(A); + return Attr.isEnumAttribute() || Attr.isIntAttribute(); +} + +LLVMBool LLVMIsStringAttribute(LLVMAttributeRef A) { + return unwrap(A).isStringAttribute(); +} + char *LLVMGetDiagInfoDescription(LLVMDiagnosticInfoRef DI) { std::string MsgStorage; raw_string_ostream Stream(MsgStorage); @@ -120,7 +188,7 @@ char *LLVMGetDiagInfoDescription(LLVMDiagnosticInfoRef DI) { return LLVMCreateMessage(MsgStorage.c_str()); } -LLVMDiagnosticSeverity LLVMGetDiagInfoSeverity(LLVMDiagnosticInfoRef DI){ +LLVMDiagnosticSeverity LLVMGetDiagInfoSeverity(LLVMDiagnosticInfoRef DI) { LLVMDiagnosticSeverity severity; switch(unwrap(DI)->getSeverity()) { @@ -141,13 +209,10 @@ LLVMDiagnosticSeverity LLVMGetDiagInfoSeverity(LLVMDiagnosticInfoRef DI){ return severity; } - - - /*===-- Operations on modules ---------------------------------------------===*/ LLVMModuleRef LLVMModuleCreateWithName(const char *ModuleID) { - return wrap(new Module(ModuleID, getGlobalContext())); + return wrap(new Module(ModuleID, *GlobalContext)); } LLVMModuleRef LLVMModuleCreateWithNameInContext(const char *ModuleID, @@ -159,13 +224,28 @@ void LLVMDisposeModule(LLVMModuleRef M) { delete unwrap(M); } +const char *LLVMGetModuleIdentifier(LLVMModuleRef M, size_t *Len) { + auto &Str = unwrap(M)->getModuleIdentifier(); + *Len = Str.length(); + return Str.c_str(); +} + +void LLVMSetModuleIdentifier(LLVMModuleRef M, const char *Ident, size_t Len) { + unwrap(M)->setModuleIdentifier(StringRef(Ident, Len)); +} + + /*--.. Data layout .........................................................--*/ -const char * LLVMGetDataLayout(LLVMModuleRef M) { +const char *LLVMGetDataLayoutStr(LLVMModuleRef M) { return unwrap(M)->getDataLayoutStr().c_str(); } -void LLVMSetDataLayout(LLVMModuleRef M, const char *Triple) { - unwrap(M)->setDataLayout(Triple); +const char *LLVMGetDataLayout(LLVMModuleRef M) { + return LLVMGetDataLayoutStr(M); +} + +void LLVMSetDataLayout(LLVMModuleRef M, const char *DataLayoutStr) { + unwrap(M)->setDataLayout(DataLayoutStr); } /*--.. Target triple .......................................................--*/ @@ -537,6 +617,17 @@ LLVMTypeRef LLVMTypeOf(LLVMValueRef Val) { return wrap(unwrap(Val)->getType()); } +LLVMValueKind LLVMGetValueKind(LLVMValueRef Val) { + switch(unwrap(Val)->getValueID()) { +#define HANDLE_VALUE(Name) \ + case Value::Name##Val: \ + return LLVM##Name##ValueKind; +#include "llvm/IR/Value.def" + default: + return LLVMInstructionValueKind; + } +} + const char *LLVMGetValueName(LLVMValueRef Val) { return unwrap(Val)->getName().data(); } @@ -719,8 +810,7 @@ LLVMBool LLVMIsUndef(LLVMValueRef Val) { } LLVMValueRef LLVMConstPointerNull(LLVMTypeRef Ty) { - return - wrap(ConstantPointerNull::get(unwrap<PointerType>(Ty))); + return wrap(ConstantPointerNull::get(unwrap<PointerType>(Ty))); } /*--.. Operations on metadata nodes ........................................--*/ @@ -767,26 +857,24 @@ LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count) { return LLVMMDNodeInContext(LLVMGetGlobalContext(), Vals, Count); } -const char *LLVMGetMDString(LLVMValueRef V, unsigned* Len) { +const char *LLVMGetMDString(LLVMValueRef V, unsigned *Length) { if (const auto *MD = dyn_cast<MetadataAsValue>(unwrap(V))) if (const MDString *S = dyn_cast<MDString>(MD->getMetadata())) { - *Len = S->getString().size(); + *Length = S->getString().size(); return S->getString().data(); } - *Len = 0; + *Length = 0; return nullptr; } -unsigned LLVMGetMDNodeNumOperands(LLVMValueRef V) -{ +unsigned LLVMGetMDNodeNumOperands(LLVMValueRef V) { auto *MD = cast<MetadataAsValue>(unwrap(V)); if (isa<ValueAsMetadata>(MD->getMetadata())) return 1; return cast<MDNode>(MD->getMetadata())->getNumOperands(); } -void LLVMGetMDNodeOperands(LLVMValueRef V, LLVMValueRef *Dest) -{ +void LLVMGetMDNodeOperands(LLVMValueRef V, LLVMValueRef *Dest) { auto *MD = cast<MetadataAsValue>(unwrap(V)); if (auto *MDV = dyn_cast<ValueAsMetadata>(MD->getMetadata())) { *Dest = wrap(MDV->getValue()); @@ -799,17 +887,16 @@ void LLVMGetMDNodeOperands(LLVMValueRef V, LLVMValueRef *Dest) Dest[i] = getMDNodeOperandImpl(Context, N, i); } -unsigned LLVMGetNamedMetadataNumOperands(LLVMModuleRef M, const char* name) -{ - if (NamedMDNode *N = unwrap(M)->getNamedMetadata(name)) { +unsigned LLVMGetNamedMetadataNumOperands(LLVMModuleRef M, const char *Name) { + if (NamedMDNode *N = unwrap(M)->getNamedMetadata(Name)) { return N->getNumOperands(); } return 0; } -void LLVMGetNamedMetadataOperands(LLVMModuleRef M, const char* name, LLVMValueRef *Dest) -{ - NamedMDNode *N = unwrap(M)->getNamedMetadata(name); +void LLVMGetNamedMetadataOperands(LLVMModuleRef M, const char *Name, + LLVMValueRef *Dest) { + NamedMDNode *N = unwrap(M)->getNamedMetadata(Name); if (!N) return; LLVMContext &Context = unwrap(M)->getContext(); @@ -817,10 +904,9 @@ void LLVMGetNamedMetadataOperands(LLVMModuleRef M, const char* name, LLVMValueRe Dest[i] = wrap(MetadataAsValue::get(Context, N->getOperand(i))); } -void LLVMAddNamedMetadataOperand(LLVMModuleRef M, const char* name, - LLVMValueRef Val) -{ - NamedMDNode *N = unwrap(M)->getOrInsertNamedMetadata(name); +void LLVMAddNamedMetadataOperand(LLVMModuleRef M, const char *Name, + LLVMValueRef Val) { + NamedMDNode *N = unwrap(M)->getOrInsertNamedMetadata(Name); if (!N) return; if (!Val) @@ -908,13 +994,6 @@ LLVMValueRef LLVMConstStringInContext(LLVMContextRef C, const char *Str, return wrap(ConstantDataArray::getString(*unwrap(C), StringRef(Str, Length), DontNullTerminate == 0)); } -LLVMValueRef LLVMConstStructInContext(LLVMContextRef C, - LLVMValueRef *ConstantVals, - unsigned Count, LLVMBool Packed) { - Constant **Elements = unwrap<Constant>(ConstantVals, Count); - return wrap(ConstantStruct::getAnon(*unwrap(C), makeArrayRef(Elements, Count), - Packed != 0)); -} LLVMValueRef LLVMConstString(const char *Str, unsigned Length, LLVMBool DontNullTerminate) { @@ -922,18 +1001,18 @@ LLVMValueRef LLVMConstString(const char *Str, unsigned Length, DontNullTerminate); } -LLVMValueRef LLVMGetElementAsConstant(LLVMValueRef c, unsigned idx) { - return wrap(static_cast<ConstantDataSequential*>(unwrap(c))->getElementAsConstant(idx)); +LLVMValueRef LLVMGetElementAsConstant(LLVMValueRef C, unsigned idx) { + return wrap(unwrap<ConstantDataSequential>(C)->getElementAsConstant(idx)); } -LLVMBool LLVMIsConstantString(LLVMValueRef c) { - return static_cast<ConstantDataSequential*>(unwrap(c))->isString(); +LLVMBool LLVMIsConstantString(LLVMValueRef C) { + return unwrap<ConstantDataSequential>(C)->isString(); } -const char *LLVMGetAsString(LLVMValueRef c, size_t* Length) { - StringRef str = static_cast<ConstantDataSequential*>(unwrap(c))->getAsString(); - *Length = str.size(); - return str.data(); +const char *LLVMGetAsString(LLVMValueRef C, size_t *Length) { + StringRef Str = unwrap<ConstantDataSequential>(C)->getAsString(); + *Length = Str.size(); + return Str.data(); } LLVMValueRef LLVMConstArray(LLVMTypeRef ElementTy, @@ -942,6 +1021,14 @@ LLVMValueRef LLVMConstArray(LLVMTypeRef ElementTy, return wrap(ConstantArray::get(ArrayType::get(unwrap(ElementTy), Length), V)); } +LLVMValueRef LLVMConstStructInContext(LLVMContextRef C, + LLVMValueRef *ConstantVals, + unsigned Count, LLVMBool Packed) { + Constant **Elements = unwrap<Constant>(ConstantVals, Count); + return wrap(ConstantStruct::getAnon(*unwrap(C), makeArrayRef(Elements, Count), + Packed != 0)); +} + LLVMValueRef LLVMConstStruct(LLVMValueRef *ConstantVals, unsigned Count, LLVMBool Packed) { return LLVMConstStructInContext(LLVMGetGlobalContext(), ConstantVals, Count, @@ -1441,7 +1528,9 @@ void LLVMSetLinkage(LLVMValueRef Global, LLVMLinkage Linkage) { } const char *LLVMGetSection(LLVMValueRef Global) { - return unwrap<GlobalValue>(Global)->getSection(); + // Using .data() is safe because of how GlobalObject::setSection is + // implemented. + return unwrap<GlobalValue>(Global)->getSection().data(); } void LLVMSetSection(LLVMValueRef Global, const char *Section) { @@ -1469,11 +1558,13 @@ void LLVMSetDLLStorageClass(LLVMValueRef Global, LLVMDLLStorageClass Class) { } LLVMBool LLVMHasUnnamedAddr(LLVMValueRef Global) { - return unwrap<GlobalValue>(Global)->hasUnnamedAddr(); + return unwrap<GlobalValue>(Global)->hasGlobalUnnamedAddr(); } void LLVMSetUnnamedAddr(LLVMValueRef Global, LLVMBool HasUnnamedAddr) { - unwrap<GlobalValue>(Global)->setUnnamedAddr(HasUnnamedAddr); + unwrap<GlobalValue>(Global)->setUnnamedAddr( + HasUnnamedAddr ? GlobalValue::UnnamedAddr::Global + : GlobalValue::UnnamedAddr::None); } /*--.. Operations on global variables, load and store instructions .........--*/ @@ -1697,6 +1788,10 @@ void LLVMDeleteFunction(LLVMValueRef Fn) { unwrap<Function>(Fn)->eraseFromParent(); } +LLVMBool LLVMHasPersonalityFn(LLVMValueRef Fn) { + return unwrap<Function>(Fn)->hasPersonalityFn(); +} + LLVMValueRef LLVMGetPersonalityFn(LLVMValueRef Fn) { return wrap(unwrap<Function>(Fn)->getPersonalityFn()); } @@ -1744,6 +1839,34 @@ void LLVMAddFunctionAttr(LLVMValueRef Fn, LLVMAttribute PA) { Func->setAttributes(PALnew); } +void LLVMAddAttributeAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx, + LLVMAttributeRef A) { + unwrap<Function>(F)->addAttribute(Idx, unwrap(A)); +} + +LLVMAttributeRef LLVMGetEnumAttributeAtIndex(LLVMValueRef F, + LLVMAttributeIndex Idx, + unsigned KindID) { + return wrap(unwrap<Function>(F)->getAttribute(Idx, + (Attribute::AttrKind)KindID)); +} + +LLVMAttributeRef LLVMGetStringAttributeAtIndex(LLVMValueRef F, + LLVMAttributeIndex Idx, + const char *K, unsigned KLen) { + return wrap(unwrap<Function>(F)->getAttribute(Idx, StringRef(K, KLen))); +} + +void LLVMRemoveEnumAttributeAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx, + unsigned KindID) { + unwrap<Function>(F)->removeAttribute(Idx, (Attribute::AttrKind)KindID); +} + +void LLVMRemoveStringAttributeAtIndex(LLVMValueRef F, LLVMAttributeIndex Idx, + const char *K, unsigned KLen) { + unwrap<Function>(F)->removeAttribute(Idx, StringRef(K, KLen)); +} + void LLVMAddTargetDependentFunctionAttr(LLVMValueRef Fn, const char *A, const char *V) { Function *Func = unwrap<Function>(Fn); @@ -1849,7 +1972,6 @@ LLVMAttribute LLVMGetAttribute(LLVMValueRef Arg) { Raw(A->getArgNo()+1); } - void LLVMSetParamAlignment(LLVMValueRef Arg, unsigned align) { Argument *A = unwrap<Argument>(Arg); AttrBuilder B; @@ -1871,6 +1993,10 @@ LLVMBasicBlockRef LLVMValueAsBasicBlock(LLVMValueRef Val) { return wrap(unwrap<BasicBlock>(Val)); } +const char *LLVMGetBasicBlockName(LLVMBasicBlockRef BB) { + return unwrap(BB)->getName().data(); +} + LLVMValueRef LLVMGetBasicBlockParent(LLVMBasicBlockRef BB) { return wrap(unwrap(BB)->getParent()); } @@ -1885,8 +2011,8 @@ unsigned LLVMCountBasicBlocks(LLVMValueRef FnRef) { void LLVMGetBasicBlocks(LLVMValueRef FnRef, LLVMBasicBlockRef *BasicBlocksRefs){ Function *Fn = unwrap<Function>(FnRef); - for (Function::iterator I = Fn->begin(), E = Fn->end(); I != E; I++) - *BasicBlocksRefs++ = wrap(&*I); + for (BasicBlock &BB : *Fn) + *BasicBlocksRefs++ = wrap(&BB); } LLVMBasicBlockRef LLVMGetEntryBasicBlock(LLVMValueRef Fn) { @@ -2001,6 +2127,10 @@ LLVMValueRef LLVMGetPreviousInstruction(LLVMValueRef Inst) { return wrap(&*--I); } +void LLVMInstructionRemoveFromParent(LLVMValueRef Inst) { + unwrap<Instruction>(Inst)->removeFromParent(); +} + void LLVMInstructionEraseFromParent(LLVMValueRef Inst) { unwrap<Instruction>(Inst)->eraseFromParent(); } @@ -2037,22 +2167,17 @@ LLVMValueRef LLVMInstructionClone(LLVMValueRef Inst) { /*--.. Call and invoke instructions ........................................--*/ +unsigned LLVMGetNumArgOperands(LLVMValueRef Instr) { + return CallSite(unwrap<Instruction>(Instr)).getNumArgOperands(); +} + unsigned LLVMGetInstructionCallConv(LLVMValueRef Instr) { - Value *V = unwrap(Instr); - if (CallInst *CI = dyn_cast<CallInst>(V)) - return CI->getCallingConv(); - if (InvokeInst *II = dyn_cast<InvokeInst>(V)) - return II->getCallingConv(); - llvm_unreachable("LLVMGetInstructionCallConv applies only to call and invoke!"); + return CallSite(unwrap<Instruction>(Instr)).getCallingConv(); } void LLVMSetInstructionCallConv(LLVMValueRef Instr, unsigned CC) { - Value *V = unwrap(Instr); - if (CallInst *CI = dyn_cast<CallInst>(V)) - return CI->setCallingConv(static_cast<CallingConv::ID>(CC)); - else if (InvokeInst *II = dyn_cast<InvokeInst>(V)) - return II->setCallingConv(static_cast<CallingConv::ID>(CC)); - llvm_unreachable("LLVMSetInstructionCallConv applies only to call and invoke!"); + return CallSite(unwrap<Instruction>(Instr)) + .setCallingConv(static_cast<CallingConv::ID>(CC)); } void LLVMAddInstrAttribute(LLVMValueRef Instr, unsigned index, @@ -2086,6 +2211,40 @@ void LLVMSetInstrParamAlignment(LLVMValueRef Instr, unsigned index, index, B))); } +void LLVMAddCallSiteAttribute(LLVMValueRef C, LLVMAttributeIndex Idx, + LLVMAttributeRef A) { + CallSite(unwrap<Instruction>(C)).addAttribute(Idx, unwrap(A)); +} + +LLVMAttributeRef LLVMGetCallSiteEnumAttribute(LLVMValueRef C, + LLVMAttributeIndex Idx, + unsigned KindID) { + return wrap(CallSite(unwrap<Instruction>(C)) + .getAttribute(Idx, (Attribute::AttrKind)KindID)); +} + +LLVMAttributeRef LLVMGetCallSiteStringAttribute(LLVMValueRef C, + LLVMAttributeIndex Idx, + const char *K, unsigned KLen) { + return wrap(CallSite(unwrap<Instruction>(C)) + .getAttribute(Idx, StringRef(K, KLen))); +} + +void LLVMRemoveCallSiteEnumAttribute(LLVMValueRef C, LLVMAttributeIndex Idx, + unsigned KindID) { + CallSite(unwrap<Instruction>(C)) + .removeAttribute(Idx, (Attribute::AttrKind)KindID); +} + +void LLVMRemoveCallSiteStringAttribute(LLVMValueRef C, LLVMAttributeIndex Idx, + const char *K, unsigned KLen) { + CallSite(unwrap<Instruction>(C)).removeAttribute(Idx, StringRef(K, KLen)); +} + +LLVMValueRef LLVMGetCalledValue(LLVMValueRef Instr) { + return wrap(CallSite(unwrap<Instruction>(Instr)).getCalledValue()); +} + /*--.. Operations on call instructions (only) ..............................--*/ LLVMBool LLVMIsTailCall(LLVMValueRef Call) { @@ -2096,6 +2255,24 @@ void LLVMSetTailCall(LLVMValueRef Call, LLVMBool isTailCall) { unwrap<CallInst>(Call)->setTailCall(isTailCall); } +/*--.. Operations on invoke instructions (only) ............................--*/ + +LLVMBasicBlockRef LLVMGetNormalDest(LLVMValueRef Invoke) { + return wrap(unwrap<InvokeInst>(Invoke)->getNormalDest()); +} + +LLVMBasicBlockRef LLVMGetUnwindDest(LLVMValueRef Invoke) { + return wrap(unwrap<InvokeInst>(Invoke)->getUnwindDest()); +} + +void LLVMSetNormalDest(LLVMValueRef Invoke, LLVMBasicBlockRef B) { + unwrap<InvokeInst>(Invoke)->setNormalDest(unwrap(B)); +} + +void LLVMSetUnwindDest(LLVMValueRef Invoke, LLVMBasicBlockRef B) { + unwrap<InvokeInst>(Invoke)->setUnwindDest(unwrap(B)); +} + /*--.. Operations on terminators ...........................................--*/ unsigned LLVMGetNumSuccessors(LLVMValueRef Term) { @@ -2130,6 +2307,22 @@ LLVMBasicBlockRef LLVMGetSwitchDefaultDest(LLVMValueRef Switch) { return wrap(unwrap<SwitchInst>(Switch)->getDefaultDest()); } +/*--.. Operations on alloca instructions (only) ............................--*/ + +LLVMTypeRef LLVMGetAllocatedType(LLVMValueRef Alloca) { + return wrap(unwrap<AllocaInst>(Alloca)->getAllocatedType()); +} + +/*--.. Operations on gep instructions (only) ...............................--*/ + +LLVMBool LLVMIsInBounds(LLVMValueRef GEP) { + return unwrap<GetElementPtrInst>(GEP)->isInBounds(); +} + +void LLVMSetIsInBounds(LLVMValueRef GEP, LLVMBool InBounds) { + return unwrap<GetElementPtrInst>(GEP)->setIsInBounds(InBounds); +} + /*--.. Operations on phi nodes .............................................--*/ void LLVMAddIncoming(LLVMValueRef PhiNode, LLVMValueRef *IncomingValues, @@ -2151,6 +2344,30 @@ LLVMBasicBlockRef LLVMGetIncomingBlock(LLVMValueRef PhiNode, unsigned Index) { return wrap(unwrap<PHINode>(PhiNode)->getIncomingBlock(Index)); } +/*--.. Operations on extractvalue and insertvalue nodes ....................--*/ + +unsigned LLVMGetNumIndices(LLVMValueRef Inst) { + auto *I = unwrap(Inst); + if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) + return GEP->getNumIndices(); + if (auto *EV = dyn_cast<ExtractValueInst>(I)) + return EV->getNumIndices(); + if (auto *IV = dyn_cast<InsertValueInst>(I)) + return IV->getNumIndices(); + llvm_unreachable( + "LLVMGetNumIndices applies only to extractvalue and insertvalue!"); +} + +const unsigned *LLVMGetIndices(LLVMValueRef Inst) { + auto *I = unwrap(Inst); + if (auto *EV = dyn_cast<ExtractValueInst>(I)) + return EV->getIndices().data(); + if (auto *IV = dyn_cast<InsertValueInst>(I)) + return IV->getIndices().data(); + llvm_unreachable( + "LLVMGetIndices applies only to extractvalue and insertvalue!"); +} + /*===-- Instruction builders ----------------------------------------------===*/ @@ -2291,11 +2508,23 @@ void LLVMAddDestination(LLVMValueRef IndirectBr, LLVMBasicBlockRef Dest) { unwrap<IndirectBrInst>(IndirectBr)->addDestination(unwrap(Dest)); } +unsigned LLVMGetNumClauses(LLVMValueRef LandingPad) { + return unwrap<LandingPadInst>(LandingPad)->getNumClauses(); +} + +LLVMValueRef LLVMGetClause(LLVMValueRef LandingPad, unsigned Idx) { + return wrap(unwrap<LandingPadInst>(LandingPad)->getClause(Idx)); +} + void LLVMAddClause(LLVMValueRef LandingPad, LLVMValueRef ClauseVal) { unwrap<LandingPadInst>(LandingPad)-> addClause(cast<Constant>(unwrap(ClauseVal))); } +LLVMBool LLVMIsCleanup(LLVMValueRef LandingPad) { + return unwrap<LandingPadInst>(LandingPad)->isCleanup(); +} + void LLVMSetCleanup(LLVMValueRef LandingPad, LLVMBool Val) { unwrap<LandingPadInst>(LandingPad)->setCleanup(Val); } @@ -2507,14 +2736,15 @@ LLVMValueRef LLVMBuildStore(LLVMBuilderRef B, LLVMValueRef Val, static AtomicOrdering mapFromLLVMOrdering(LLVMAtomicOrdering Ordering) { switch (Ordering) { - case LLVMAtomicOrderingNotAtomic: return NotAtomic; - case LLVMAtomicOrderingUnordered: return Unordered; - case LLVMAtomicOrderingMonotonic: return Monotonic; - case LLVMAtomicOrderingAcquire: return Acquire; - case LLVMAtomicOrderingRelease: return Release; - case LLVMAtomicOrderingAcquireRelease: return AcquireRelease; + case LLVMAtomicOrderingNotAtomic: return AtomicOrdering::NotAtomic; + case LLVMAtomicOrderingUnordered: return AtomicOrdering::Unordered; + case LLVMAtomicOrderingMonotonic: return AtomicOrdering::Monotonic; + case LLVMAtomicOrderingAcquire: return AtomicOrdering::Acquire; + case LLVMAtomicOrderingRelease: return AtomicOrdering::Release; + case LLVMAtomicOrderingAcquireRelease: + return AtomicOrdering::AcquireRelease; case LLVMAtomicOrderingSequentiallyConsistent: - return SequentiallyConsistent; + return AtomicOrdering::SequentiallyConsistent; } llvm_unreachable("Invalid LLVMAtomicOrdering value!"); @@ -2522,13 +2752,14 @@ static AtomicOrdering mapFromLLVMOrdering(LLVMAtomicOrdering Ordering) { static LLVMAtomicOrdering mapToLLVMOrdering(AtomicOrdering Ordering) { switch (Ordering) { - case NotAtomic: return LLVMAtomicOrderingNotAtomic; - case Unordered: return LLVMAtomicOrderingUnordered; - case Monotonic: return LLVMAtomicOrderingMonotonic; - case Acquire: return LLVMAtomicOrderingAcquire; - case Release: return LLVMAtomicOrderingRelease; - case AcquireRelease: return LLVMAtomicOrderingAcquireRelease; - case SequentiallyConsistent: + case AtomicOrdering::NotAtomic: return LLVMAtomicOrderingNotAtomic; + case AtomicOrdering::Unordered: return LLVMAtomicOrderingUnordered; + case AtomicOrdering::Monotonic: return LLVMAtomicOrderingMonotonic; + case AtomicOrdering::Acquire: return LLVMAtomicOrderingAcquire; + case AtomicOrdering::Release: return LLVMAtomicOrderingRelease; + case AtomicOrdering::AcquireRelease: + return LLVMAtomicOrderingAcquireRelease; + case AtomicOrdering::SequentiallyConsistent: return LLVMAtomicOrderingSequentiallyConsistent; } @@ -2824,6 +3055,61 @@ LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B,LLVMAtomicRMWBinOp op, mapFromLLVMOrdering(ordering), singleThread ? SingleThread : CrossThread)); } +LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef Ptr, + LLVMValueRef Cmp, LLVMValueRef New, + LLVMAtomicOrdering SuccessOrdering, + LLVMAtomicOrdering FailureOrdering, + LLVMBool singleThread) { + + return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(Ptr), unwrap(Cmp), + unwrap(New), mapFromLLVMOrdering(SuccessOrdering), + mapFromLLVMOrdering(FailureOrdering), + singleThread ? SingleThread : CrossThread)); +} + + +LLVMBool LLVMIsAtomicSingleThread(LLVMValueRef AtomicInst) { + Value *P = unwrap<Value>(AtomicInst); + + if (AtomicRMWInst *I = dyn_cast<AtomicRMWInst>(P)) + return I->getSynchScope() == SingleThread; + return cast<AtomicCmpXchgInst>(P)->getSynchScope() == SingleThread; +} + +void LLVMSetAtomicSingleThread(LLVMValueRef AtomicInst, LLVMBool NewValue) { + Value *P = unwrap<Value>(AtomicInst); + SynchronizationScope Sync = NewValue ? SingleThread : CrossThread; + + if (AtomicRMWInst *I = dyn_cast<AtomicRMWInst>(P)) + return I->setSynchScope(Sync); + return cast<AtomicCmpXchgInst>(P)->setSynchScope(Sync); +} + +LLVMAtomicOrdering LLVMGetCmpXchgSuccessOrdering(LLVMValueRef CmpXchgInst) { + Value *P = unwrap<Value>(CmpXchgInst); + return mapToLLVMOrdering(cast<AtomicCmpXchgInst>(P)->getSuccessOrdering()); +} + +void LLVMSetCmpXchgSuccessOrdering(LLVMValueRef CmpXchgInst, + LLVMAtomicOrdering Ordering) { + Value *P = unwrap<Value>(CmpXchgInst); + AtomicOrdering O = mapFromLLVMOrdering(Ordering); + + return cast<AtomicCmpXchgInst>(P)->setSuccessOrdering(O); +} + +LLVMAtomicOrdering LLVMGetCmpXchgFailureOrdering(LLVMValueRef CmpXchgInst) { + Value *P = unwrap<Value>(CmpXchgInst); + return mapToLLVMOrdering(cast<AtomicCmpXchgInst>(P)->getFailureOrdering()); +} + +void LLVMSetCmpXchgFailureOrdering(LLVMValueRef CmpXchgInst, + LLVMAtomicOrdering Ordering) { + Value *P = unwrap<Value>(CmpXchgInst); + AtomicOrdering O = mapFromLLVMOrdering(Ordering); + + return cast<AtomicCmpXchgInst>(P)->setFailureOrdering(O); +} /*===-- Module providers --------------------------------------------------===*/ diff --git a/lib/IR/DIBuilder.cpp b/lib/IR/DIBuilder.cpp index b7841fe2b85cc..01b47f386e1c0 100644 --- a/lib/IR/DIBuilder.cpp +++ b/lib/IR/DIBuilder.cpp @@ -19,44 +19,11 @@ #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Dwarf.h" +#include "LLVMContextImpl.h" using namespace llvm; using namespace llvm::dwarf; -namespace { -class HeaderBuilder { - /// \brief Whether there are any fields yet. - /// - /// Note that this is not equivalent to \c Chars.empty(), since \a concat() - /// may have been called already with an empty string. - bool IsEmpty; - SmallVector<char, 256> Chars; - -public: - HeaderBuilder() : IsEmpty(true) {} - HeaderBuilder(const HeaderBuilder &X) : IsEmpty(X.IsEmpty), Chars(X.Chars) {} - HeaderBuilder(HeaderBuilder &&X) - : IsEmpty(X.IsEmpty), Chars(std::move(X.Chars)) {} - - template <class Twineable> HeaderBuilder &concat(Twineable &&X) { - if (IsEmpty) - IsEmpty = false; - else - Chars.push_back(0); - Twine(X).toVector(Chars); - return *this; - } - - MDString *get(LLVMContext &Context) const { - return MDString::get(Context, StringRef(Chars.begin(), Chars.size())); - } - - static HeaderBuilder get(unsigned Tag) { - return HeaderBuilder().concat("0x" + Twine::utohexstr(Tag)); - } -}; -} - DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes) : M(m), VMContext(M.getContext()), CUNode(nullptr), DeclareFn(nullptr), ValueFn(nullptr), @@ -95,17 +62,25 @@ void DIBuilder::finalize() { CUNode->replaceRetainedTypes(MDTuple::get(VMContext, RetainValues)); DISubprogramArray SPs = MDTuple::get(VMContext, AllSubprograms); - if (!AllSubprograms.empty()) - CUNode->replaceSubprograms(SPs.get()); - - for (auto *SP : SPs) { - if (MDTuple *Temp = SP->getVariables().get()) { - const auto &PV = PreservedVariables.lookup(SP); - SmallVector<Metadata *, 4> Variables(PV.begin(), PV.end()); - DINodeArray AV = getOrCreateArray(Variables); - TempMDTuple(Temp)->replaceAllUsesWith(AV.get()); - } - } + auto resolveVariables = [&](DISubprogram *SP) { + MDTuple *Temp = SP->getVariables().get(); + if (!Temp) + return; + + SmallVector<Metadata *, 4> Variables; + + auto PV = PreservedVariables.find(SP); + if (PV != PreservedVariables.end()) + Variables.append(PV->second.begin(), PV->second.end()); + + DINodeArray AV = getOrCreateArray(Variables); + TempMDTuple(Temp)->replaceAllUsesWith(AV.get()); + }; + for (auto *SP : SPs) + resolveVariables(SP); + for (auto *N : RetainValues) + if (auto *SP = dyn_cast<DISubprogram>(N)) + resolveVariables(SP); if (!AllGVs.empty()) CUNode->replaceGlobalVariables(MDTuple::get(VMContext, AllGVs)); @@ -136,7 +111,7 @@ static DIScope *getNonCompileUnitScope(DIScope *N) { DICompileUnit *DIBuilder::createCompileUnit( unsigned Lang, StringRef Filename, StringRef Directory, StringRef Producer, bool isOptimized, StringRef Flags, unsigned RunTimeVer, StringRef SplitName, - DebugEmissionKind Kind, uint64_t DWOId, bool EmitDebugInfo) { + DICompileUnit::DebugEmissionKind Kind, uint64_t DWOId) { assert(((Lang <= dwarf::DW_LANG_Fortran08 && Lang >= dwarf::DW_LANG_C89) || (Lang <= dwarf::DW_LANG_hi_user && Lang >= dwarf::DW_LANG_lo_user)) && @@ -147,19 +122,12 @@ DICompileUnit *DIBuilder::createCompileUnit( assert(!CUNode && "Can only make one compile unit per DIBuilder instance"); CUNode = DICompileUnit::getDistinct( VMContext, Lang, DIFile::get(VMContext, Filename, Directory), Producer, - isOptimized, Flags, RunTimeVer, SplitName, Kind, nullptr, - nullptr, nullptr, nullptr, nullptr, nullptr, DWOId); + isOptimized, Flags, RunTimeVer, SplitName, Kind, nullptr, nullptr, + nullptr, nullptr, nullptr, DWOId); // Create a named metadata so that it is easier to find cu in a module. - // Note that we only generate this when the caller wants to actually - // emit debug information. When we are only interested in tracking - // source line locations throughout the backend, we prevent codegen from - // emitting debug info in the final output by not generating llvm.dbg.cu. - if (EmitDebugInfo) { - NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.cu"); - NMD->addOperand(CUNode); - } - + NamedMDNode *NMD = M.getOrInsertNamedMetadata("llvm.dbg.cu"); + NMD->addOperand(CUNode); trackIfUnresolved(CUNode); return CUNode; } @@ -168,8 +136,12 @@ static DIImportedEntity * createImportedModule(LLVMContext &C, dwarf::Tag Tag, DIScope *Context, Metadata *NS, unsigned Line, StringRef Name, SmallVectorImpl<TrackingMDNodeRef> &AllImportedModules) { + unsigned EntitiesCount = C.pImpl->DIImportedEntitys.size(); auto *M = DIImportedEntity::get(C, Tag, Context, DINodeRef(NS), Line, Name); - AllImportedModules.emplace_back(M); + if (EntitiesCount < C.pImpl->DIImportedEntitys.size()) + // A new Imported Entity was just added to the context. + // Add it to the Imported Modules list. + AllImportedModules.emplace_back(M); return M; } @@ -200,8 +172,7 @@ DIImportedEntity *DIBuilder::createImportedDeclaration(DIScope *Context, // Make sure to use the unique identifier based metadata reference for // types that have one. return ::createImportedModule(VMContext, dwarf::DW_TAG_imported_declaration, - Context, DINodeRef::get(Decl), Line, Name, - AllImportedModules); + Context, Decl, Line, Name, AllImportedModules); } DIFile *DIBuilder::createFile(StringRef Filename, StringRef Directory) { @@ -231,8 +202,8 @@ DIBasicType *DIBuilder::createBasicType(StringRef Name, uint64_t SizeInBits, } DIDerivedType *DIBuilder::createQualifiedType(unsigned Tag, DIType *FromTy) { - return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, - DITypeRef::get(FromTy), 0, 0, 0, 0); + return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, FromTy, 0, + 0, 0, 0); } DIDerivedType *DIBuilder::createPointerType(DIType *PointeeTy, @@ -241,43 +212,41 @@ DIDerivedType *DIBuilder::createPointerType(DIType *PointeeTy, StringRef Name) { // FIXME: Why is there a name here? return DIDerivedType::get(VMContext, dwarf::DW_TAG_pointer_type, Name, - nullptr, 0, nullptr, DITypeRef::get(PointeeTy), - SizeInBits, AlignInBits, 0, 0); + nullptr, 0, nullptr, PointeeTy, SizeInBits, + AlignInBits, 0, 0); } DIDerivedType *DIBuilder::createMemberPointerType(DIType *PointeeTy, DIType *Base, uint64_t SizeInBits, - uint64_t AlignInBits) { + uint64_t AlignInBits, + unsigned Flags) { return DIDerivedType::get(VMContext, dwarf::DW_TAG_ptr_to_member_type, "", - nullptr, 0, nullptr, DITypeRef::get(PointeeTy), - SizeInBits, AlignInBits, 0, 0, - DITypeRef::get(Base)); + nullptr, 0, nullptr, PointeeTy, SizeInBits, + AlignInBits, 0, Flags, Base); } DIDerivedType *DIBuilder::createReferenceType(unsigned Tag, DIType *RTy, uint64_t SizeInBits, uint64_t AlignInBits) { assert(RTy && "Unable to create reference type"); - return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, - DITypeRef::get(RTy), SizeInBits, AlignInBits, 0, 0); + return DIDerivedType::get(VMContext, Tag, "", nullptr, 0, nullptr, RTy, + SizeInBits, AlignInBits, 0, 0); } DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name, DIFile *File, unsigned LineNo, DIScope *Context) { return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File, - LineNo, - DIScopeRef::get(getNonCompileUnitScope(Context)), - DITypeRef::get(Ty), 0, 0, 0, 0); + LineNo, getNonCompileUnitScope(Context), Ty, 0, 0, + 0, 0); } DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) { assert(Ty && "Invalid type!"); assert(FriendTy && "Invalid friend type!"); - return DIDerivedType::get(VMContext, dwarf::DW_TAG_friend, "", nullptr, 0, - DITypeRef::get(Ty), DITypeRef::get(FriendTy), 0, 0, - 0, 0); + return DIDerivedType::get(VMContext, dwarf::DW_TAG_friend, "", nullptr, 0, Ty, + FriendTy, 0, 0, 0, 0); } DIDerivedType *DIBuilder::createInheritance(DIType *Ty, DIType *BaseTy, @@ -285,8 +254,7 @@ DIDerivedType *DIBuilder::createInheritance(DIType *Ty, DIType *BaseTy, unsigned Flags) { assert(Ty && "Unable to create inheritance"); return DIDerivedType::get(VMContext, dwarf::DW_TAG_inheritance, "", nullptr, - 0, DITypeRef::get(Ty), DITypeRef::get(BaseTy), 0, 0, - BaseOffset, Flags); + 0, Ty, BaseTy, 0, 0, BaseOffset, Flags); } DIDerivedType *DIBuilder::createMemberType(DIScope *Scope, StringRef Name, @@ -295,10 +263,9 @@ DIDerivedType *DIBuilder::createMemberType(DIScope *Scope, StringRef Name, uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, DIType *Ty) { - return DIDerivedType::get( - VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, - DIScopeRef::get(getNonCompileUnitScope(Scope)), DITypeRef::get(Ty), - SizeInBits, AlignInBits, OffsetInBits, Flags); + return DIDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File, + LineNumber, getNonCompileUnitScope(Scope), Ty, + SizeInBits, AlignInBits, OffsetInBits, Flags); } static ConstantAsMetadata *getConstantOrNull(Constant *C) { @@ -307,16 +274,27 @@ static ConstantAsMetadata *getConstantOrNull(Constant *C) { return nullptr; } +DIDerivedType *DIBuilder::createBitFieldMemberType( + DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, + uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, + uint64_t StorageOffsetInBits, unsigned Flags, DIType *Ty) { + Flags |= DINode::FlagBitField; + return DIDerivedType::get( + VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, + getNonCompileUnitScope(Scope), Ty, SizeInBits, AlignInBits, OffsetInBits, + Flags, ConstantAsMetadata::get(ConstantInt::get( + IntegerType::get(VMContext, 64), StorageOffsetInBits))); +} + DIDerivedType *DIBuilder::createStaticMemberType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, DIType *Ty, unsigned Flags, llvm::Constant *Val) { Flags |= DINode::FlagStaticMember; - return DIDerivedType::get( - VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, - DIScopeRef::get(getNonCompileUnitScope(Scope)), DITypeRef::get(Ty), 0, 0, - 0, Flags, getConstantOrNull(Val)); + return DIDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File, + LineNumber, getNonCompileUnitScope(Scope), Ty, 0, 0, + 0, Flags, getConstantOrNull(Val)); } DIDerivedType *DIBuilder::createObjCIVar(StringRef Name, DIFile *File, @@ -325,10 +303,10 @@ DIDerivedType *DIBuilder::createObjCIVar(StringRef Name, DIFile *File, uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, DIType *Ty, MDNode *PropertyNode) { - return DIDerivedType::get( - VMContext, dwarf::DW_TAG_member, Name, File, LineNumber, - DIScopeRef::get(getNonCompileUnitScope(File)), DITypeRef::get(Ty), - SizeInBits, AlignInBits, OffsetInBits, Flags, PropertyNode); + return DIDerivedType::get(VMContext, dwarf::DW_TAG_member, Name, File, + LineNumber, getNonCompileUnitScope(File), Ty, + SizeInBits, AlignInBits, OffsetInBits, Flags, + PropertyNode); } DIObjCProperty * @@ -336,15 +314,14 @@ DIBuilder::createObjCProperty(StringRef Name, DIFile *File, unsigned LineNumber, StringRef GetterName, StringRef SetterName, unsigned PropertyAttributes, DIType *Ty) { return DIObjCProperty::get(VMContext, Name, File, LineNumber, GetterName, - SetterName, PropertyAttributes, - DITypeRef::get(Ty)); + SetterName, PropertyAttributes, Ty); } DITemplateTypeParameter * DIBuilder::createTemplateTypeParameter(DIScope *Context, StringRef Name, DIType *Ty) { assert((!Context || isa<DICompileUnit>(Context)) && "Expected compile unit"); - return DITemplateTypeParameter::get(VMContext, Name, DITypeRef::get(Ty)); + return DITemplateTypeParameter::get(VMContext, Name, Ty); } static DITemplateValueParameter * @@ -352,8 +329,7 @@ createTemplateValueParameterHelper(LLVMContext &VMContext, unsigned Tag, DIScope *Context, StringRef Name, DIType *Ty, Metadata *MD) { assert((!Context || isa<DICompileUnit>(Context)) && "Expected compile unit"); - return DITemplateValueParameter::get(VMContext, Tag, Name, DITypeRef::get(Ty), - MD); + return DITemplateValueParameter::get(VMContext, Tag, Name, Ty, MD); } DITemplateValueParameter * @@ -390,12 +366,9 @@ DICompositeType *DIBuilder::createClassType( auto *R = DICompositeType::get( VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber, - DIScopeRef::get(getNonCompileUnitScope(Context)), - DITypeRef::get(DerivedFrom), SizeInBits, AlignInBits, OffsetInBits, Flags, - Elements, 0, DITypeRef::get(VTableHolder), + getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, + OffsetInBits, Flags, Elements, 0, VTableHolder, cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier); - if (!UniqueIdentifier.empty()) - retainType(R); trackIfUnresolved(R); return R; } @@ -407,11 +380,8 @@ DICompositeType *DIBuilder::createStructType( DIType *VTableHolder, StringRef UniqueIdentifier) { auto *R = DICompositeType::get( VMContext, dwarf::DW_TAG_structure_type, Name, File, LineNumber, - DIScopeRef::get(getNonCompileUnitScope(Context)), - DITypeRef::get(DerivedFrom), SizeInBits, AlignInBits, 0, Flags, Elements, - RunTimeLang, DITypeRef::get(VTableHolder), nullptr, UniqueIdentifier); - if (!UniqueIdentifier.empty()) - retainType(R); + getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, 0, + Flags, Elements, RunTimeLang, VTableHolder, nullptr, UniqueIdentifier); trackIfUnresolved(R); return R; } @@ -422,30 +392,23 @@ DICompositeType *DIBuilder::createUnionType( DINodeArray Elements, unsigned RunTimeLang, StringRef UniqueIdentifier) { auto *R = DICompositeType::get( VMContext, dwarf::DW_TAG_union_type, Name, File, LineNumber, - DIScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, SizeInBits, - AlignInBits, 0, Flags, Elements, RunTimeLang, nullptr, nullptr, - UniqueIdentifier); - if (!UniqueIdentifier.empty()) - retainType(R); + getNonCompileUnitScope(Scope), nullptr, SizeInBits, AlignInBits, 0, Flags, + Elements, RunTimeLang, nullptr, nullptr, UniqueIdentifier); trackIfUnresolved(R); return R; } DISubroutineType *DIBuilder::createSubroutineType(DITypeRefArray ParameterTypes, - unsigned Flags) { - return DISubroutineType::get(VMContext, Flags, ParameterTypes); + unsigned Flags, unsigned CC) { + return DISubroutineType::get(VMContext, Flags, CC, ParameterTypes); } DICompositeType *DIBuilder::createExternalTypeRef(unsigned Tag, DIFile *File, StringRef UniqueIdentifier) { assert(!UniqueIdentifier.empty() && "external type ref without uid"); - auto *CTy = - DICompositeType::get(VMContext, Tag, "", nullptr, 0, nullptr, nullptr, 0, - 0, 0, DINode::FlagExternalTypeRef, nullptr, 0, - nullptr, nullptr, UniqueIdentifier); - // Types with unique IDs need to be in the type map. - retainType(CTy); - return CTy; + return DICompositeType::get(VMContext, Tag, "", nullptr, 0, nullptr, nullptr, + 0, 0, 0, DINode::FlagExternalTypeRef, nullptr, 0, + nullptr, nullptr, UniqueIdentifier); } DICompositeType *DIBuilder::createEnumerationType( @@ -454,12 +417,9 @@ DICompositeType *DIBuilder::createEnumerationType( DIType *UnderlyingType, StringRef UniqueIdentifier) { auto *CTy = DICompositeType::get( VMContext, dwarf::DW_TAG_enumeration_type, Name, File, LineNumber, - DIScopeRef::get(getNonCompileUnitScope(Scope)), - DITypeRef::get(UnderlyingType), SizeInBits, AlignInBits, 0, 0, Elements, - 0, nullptr, nullptr, UniqueIdentifier); + getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0, + 0, Elements, 0, nullptr, nullptr, UniqueIdentifier); AllEnumTypes.push_back(CTy); - if (!UniqueIdentifier.empty()) - retainType(CTy); trackIfUnresolved(CTy); return CTy; } @@ -468,8 +428,8 @@ DICompositeType *DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits, DIType *Ty, DINodeArray Subscripts) { auto *R = DICompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", - nullptr, 0, nullptr, DITypeRef::get(Ty), Size, - AlignInBits, 0, 0, Subscripts, 0, nullptr); + nullptr, 0, nullptr, Ty, Size, AlignInBits, 0, + 0, Subscripts, 0, nullptr); trackIfUnresolved(R); return R; } @@ -477,10 +437,9 @@ DICompositeType *DIBuilder::createArrayType(uint64_t Size, uint64_t AlignInBits, DICompositeType *DIBuilder::createVectorType(uint64_t Size, uint64_t AlignInBits, DIType *Ty, DINodeArray Subscripts) { - auto *R = - DICompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", nullptr, 0, - nullptr, DITypeRef::get(Ty), Size, AlignInBits, 0, - DINode::FlagVector, Subscripts, 0, nullptr); + auto *R = DICompositeType::get(VMContext, dwarf::DW_TAG_array_type, "", + nullptr, 0, nullptr, Ty, Size, AlignInBits, 0, + DINode::FlagVector, Subscripts, 0, nullptr); trackIfUnresolved(R); return R; } @@ -507,8 +466,11 @@ DIType *DIBuilder::createObjectPointerType(DIType *Ty) { return createTypeWithFlags(VMContext, Ty, Flags); } -void DIBuilder::retainType(DIType *T) { +void DIBuilder::retainType(DIScope *T) { assert(T && "Expected non-null type"); + assert((isa<DIType>(T) || (isa<DISubprogram>(T) && + cast<DISubprogram>(T)->isDefinition() == false)) && + "Expected type or subprogram declaration"); AllRetainTypes.emplace_back(T); } @@ -522,12 +484,9 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope, // FIXME: Define in terms of createReplaceableForwardDecl() by calling // replaceWithUniqued(). auto *RetTy = DICompositeType::get( - VMContext, Tag, Name, F, Line, - DIScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, SizeInBits, - AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang, nullptr, - nullptr, UniqueIdentifier); - if (!UniqueIdentifier.empty()) - retainType(RetTy); + VMContext, Tag, Name, F, Line, getNonCompileUnitScope(Scope), nullptr, + SizeInBits, AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang, + nullptr, nullptr, UniqueIdentifier); trackIfUnresolved(RetTy); return RetTy; } @@ -536,14 +495,12 @@ DICompositeType *DIBuilder::createReplaceableCompositeType( unsigned Tag, StringRef Name, DIScope *Scope, DIFile *F, unsigned Line, unsigned RuntimeLang, uint64_t SizeInBits, uint64_t AlignInBits, unsigned Flags, StringRef UniqueIdentifier) { - auto *RetTy = DICompositeType::getTemporary( - VMContext, Tag, Name, F, Line, - DIScopeRef::get(getNonCompileUnitScope(Scope)), nullptr, - SizeInBits, AlignInBits, 0, Flags, nullptr, RuntimeLang, - nullptr, nullptr, UniqueIdentifier) - .release(); - if (!UniqueIdentifier.empty()) - retainType(RetTy); + auto *RetTy = + DICompositeType::getTemporary( + VMContext, Tag, Name, F, Line, getNonCompileUnitScope(Scope), nullptr, + SizeInBits, AlignInBits, 0, Flags, nullptr, RuntimeLang, nullptr, + nullptr, UniqueIdentifier) + .release(); trackIfUnresolved(RetTy); return RetTy; } @@ -556,7 +513,7 @@ DITypeRefArray DIBuilder::getOrCreateTypeArray(ArrayRef<Metadata *> Elements) { SmallVector<llvm::Metadata *, 16> Elts; for (unsigned i = 0, e = Elements.size(); i != e; ++i) { if (Elements[i] && isa<MDNode>(Elements[i])) - Elts.push_back(DITypeRef::get(cast<DIType>(Elements[i]))); + Elts.push_back(cast<DIType>(Elements[i])); else Elts.push_back(Elements[i]); } @@ -582,10 +539,10 @@ DIGlobalVariable *DIBuilder::createGlobalVariable( MDNode *Decl) { checkGlobalVariableScope(Context); - auto *N = DIGlobalVariable::get(VMContext, cast_or_null<DIScope>(Context), - Name, LinkageName, F, LineNumber, - DITypeRef::get(Ty), isLocalToUnit, true, Val, - cast_or_null<DIDerivedType>(Decl)); + auto *N = DIGlobalVariable::getDistinct( + VMContext, cast_or_null<DIScope>(Context), Name, LinkageName, F, + LineNumber, Ty, isLocalToUnit, true, Val, + cast_or_null<DIDerivedType>(Decl)); AllGVs.push_back(N); return N; } @@ -598,14 +555,14 @@ DIGlobalVariable *DIBuilder::createTempGlobalVariableFwdDecl( return DIGlobalVariable::getTemporary( VMContext, cast_or_null<DIScope>(Context), Name, LinkageName, F, - LineNumber, DITypeRef::get(Ty), isLocalToUnit, false, Val, + LineNumber, Ty, isLocalToUnit, false, Val, cast_or_null<DIDerivedType>(Decl)) .release(); } static DILocalVariable *createLocalVariable( LLVMContext &VMContext, - DenseMap<MDNode *, std::vector<TrackingMDNodeRef>> &PreservedVariables, + DenseMap<MDNode *, SmallVector<TrackingMDNodeRef, 1>> &PreservedVariables, DIScope *Scope, StringRef Name, unsigned ArgNo, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve, unsigned Flags) { // FIXME: Why getNonCompileUnitScope()? @@ -616,7 +573,7 @@ static DILocalVariable *createLocalVariable( auto *Node = DILocalVariable::get(VMContext, cast_or_null<DILocalScope>(Context), Name, - File, LineNo, DITypeRef::get(Ty), ArgNo, Flags); + File, LineNo, Ty, ArgNo, Flags); if (AlwaysPreserve) { // The optimizer may remove local variables. If there is an interest // to preserve variable info in such situation then stash it in a @@ -661,19 +618,6 @@ DIExpression *DIBuilder::createBitPieceExpression(unsigned OffsetInBytes, return DIExpression::get(VMContext, Addr); } -DISubprogram *DIBuilder::createFunction( - DIScopeRef Context, StringRef Name, StringRef LinkageName, DIFile *File, - unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit, - bool isDefinition, unsigned ScopeLine, unsigned Flags, bool isOptimized, - DITemplateParameterArray TParams, DISubprogram *Decl) { - // dragonegg does not generate identifier for types, so using an empty map - // to resolve the context should be fine. - DITypeIdentifierMap EmptyMap; - return createFunction(Context.resolve(EmptyMap), Name, LinkageName, File, - LineNo, Ty, isLocalToUnit, isDefinition, ScopeLine, - Flags, isOptimized, TParams, Decl); -} - template <class... Ts> static DISubprogram *getSubprogram(bool IsDistinct, Ts &&... Args) { if (IsDistinct) @@ -686,12 +630,12 @@ DISubprogram *DIBuilder::createFunction( unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit, bool isDefinition, unsigned ScopeLine, unsigned Flags, bool isOptimized, DITemplateParameterArray TParams, DISubprogram *Decl) { - auto *Node = - getSubprogram(/* IsDistinct = */ isDefinition, VMContext, - DIScopeRef::get(getNonCompileUnitScope(Context)), Name, - LinkageName, File, LineNo, Ty, isLocalToUnit, isDefinition, - ScopeLine, nullptr, 0, 0, Flags, isOptimized, TParams, Decl, - MDTuple::getTemporary(VMContext, None).release()); + auto *Node = getSubprogram( + /* IsDistinct = */ isDefinition, VMContext, + getNonCompileUnitScope(Context), Name, LinkageName, File, LineNo, Ty, + isLocalToUnit, isDefinition, ScopeLine, nullptr, 0, 0, 0, Flags, + isOptimized, isDefinition ? CUNode : nullptr, TParams, Decl, + MDTuple::getTemporary(VMContext, None).release()); if (isDefinition) AllSubprograms.push_back(Node); @@ -705,10 +649,10 @@ DISubprogram *DIBuilder::createTempFunctionFwdDecl( bool isDefinition, unsigned ScopeLine, unsigned Flags, bool isOptimized, DITemplateParameterArray TParams, DISubprogram *Decl) { return DISubprogram::getTemporary( - VMContext, DIScopeRef::get(getNonCompileUnitScope(Context)), Name, - LinkageName, File, LineNo, Ty, isLocalToUnit, isDefinition, - ScopeLine, nullptr, 0, 0, Flags, isOptimized, TParams, Decl, - nullptr) + VMContext, getNonCompileUnitScope(Context), Name, LinkageName, + File, LineNo, Ty, isLocalToUnit, isDefinition, ScopeLine, nullptr, + 0, 0, 0, Flags, isOptimized, isDefinition ? CUNode : nullptr, + TParams, Decl, nullptr) .release(); } @@ -716,17 +660,18 @@ DISubprogram * DIBuilder::createMethod(DIScope *Context, StringRef Name, StringRef LinkageName, DIFile *F, unsigned LineNo, DISubroutineType *Ty, bool isLocalToUnit, bool isDefinition, unsigned VK, - unsigned VIndex, DIType *VTableHolder, unsigned Flags, - bool isOptimized, DITemplateParameterArray TParams) { + unsigned VIndex, int ThisAdjustment, + DIType *VTableHolder, unsigned Flags, bool isOptimized, + DITemplateParameterArray TParams) { assert(getNonCompileUnitScope(Context) && "Methods should have both a Context and a context that isn't " "the compile unit."); // FIXME: Do we want to use different scope/lines? auto *SP = getSubprogram( - /* IsDistinct = */ isDefinition, VMContext, - DIScopeRef::get(cast<DIScope>(Context)), Name, LinkageName, F, LineNo, Ty, - isLocalToUnit, isDefinition, LineNo, DITypeRef::get(VTableHolder), VK, - VIndex, Flags, isOptimized, TParams, nullptr, nullptr); + /* IsDistinct = */ isDefinition, VMContext, cast<DIScope>(Context), Name, + LinkageName, F, LineNo, Ty, isLocalToUnit, isDefinition, LineNo, + VTableHolder, VK, VIndex, ThisAdjustment, Flags, isOptimized, + isDefinition ? CUNode : nullptr, TParams, nullptr, nullptr); if (isDefinition) AllSubprograms.push_back(SP); @@ -866,7 +811,7 @@ void DIBuilder::replaceVTableHolder(DICompositeType *&T, DICompositeType *VTableHolder) { { TypedTrackingMDRef<DICompositeType> N(T); - N->replaceVTableHolder(DITypeRef::get(VTableHolder)); + N->replaceVTableHolder(VTableHolder); T = N.get(); } diff --git a/lib/IR/DataLayout.cpp b/lib/IR/DataLayout.cpp index 5468f47bbfe62..20a15fb8831fb 100644 --- a/lib/IR/DataLayout.cpp +++ b/lib/IR/DataLayout.cpp @@ -52,7 +52,7 @@ StructLayout::StructLayout(StructType *ST, const DataLayout &DL) { // Add padding if necessary to align the data element properly. if ((StructSize & (TyAlign-1)) != 0) { IsPadded = true; - StructSize = RoundUpToAlignment(StructSize, TyAlign); + StructSize = alignTo(StructSize, TyAlign); } // Keep track of maximum alignment constraint. @@ -69,7 +69,7 @@ StructLayout::StructLayout(StructType *ST, const DataLayout &DL) { // and all array elements would be aligned correctly. if ((StructSize & (StructAlignment-1)) != 0) { IsPadded = true; - StructSize = RoundUpToAlignment(StructSize, StructAlignment); + StructSize = alignTo(StructSize, StructAlignment); } } @@ -718,42 +718,36 @@ Type *DataLayout::getSmallestLegalIntType(LLVMContext &C, unsigned Width) const return nullptr; } -unsigned DataLayout::getLargestLegalIntTypeSize() const { +unsigned DataLayout::getLargestLegalIntTypeSizeInBits() const { auto Max = std::max_element(LegalIntWidths.begin(), LegalIntWidths.end()); return Max != LegalIntWidths.end() ? *Max : 0; } -uint64_t DataLayout::getIndexedOffset(Type *ptrTy, - ArrayRef<Value *> Indices) const { - Type *Ty = ptrTy; - assert(Ty->isPointerTy() && "Illegal argument for getIndexedOffset()"); - uint64_t Result = 0; +int64_t DataLayout::getIndexedOffsetInType(Type *ElemTy, + ArrayRef<Value *> Indices) const { + int64_t Result = 0; + // We can use 0 as the address space as we don't need + // to get pointer types back from gep_type_iterator. + unsigned AS = 0; generic_gep_type_iterator<Value* const*> - TI = gep_type_begin(ptrTy, Indices); - for (unsigned CurIDX = 0, EndIDX = Indices.size(); CurIDX != EndIDX; - ++CurIDX, ++TI) { - if (StructType *STy = dyn_cast<StructType>(*TI)) { - assert(Indices[CurIDX]->getType() == - Type::getInt32Ty(ptrTy->getContext()) && - "Illegal struct idx"); - unsigned FieldNo = cast<ConstantInt>(Indices[CurIDX])->getZExtValue(); + GTI = gep_type_begin(ElemTy, AS, Indices), + GTE = gep_type_end(ElemTy, AS, Indices); + for (; GTI != GTE; ++GTI) { + Value *Idx = GTI.getOperand(); + if (auto *STy = dyn_cast<StructType>(*GTI)) { + assert(Idx->getType()->isIntegerTy(32) && "Illegal struct idx"); + unsigned FieldNo = cast<ConstantInt>(Idx)->getZExtValue(); // Get structure layout information... const StructLayout *Layout = getStructLayout(STy); // Add in the offset, as calculated by the structure layout info... Result += Layout->getElementOffset(FieldNo); - - // Update Ty to refer to current element - Ty = STy->getElementType(FieldNo); } else { - // Update Ty to refer to current element - Ty = cast<SequentialType>(Ty)->getElementType(); - // Get the array index and the size of each array element. - if (int64_t arrayIdx = cast<ConstantInt>(Indices[CurIDX])->getSExtValue()) - Result += (uint64_t)arrayIdx * getTypeAllocSize(Ty); + if (int64_t arrayIdx = cast<ConstantInt>(Idx)->getSExtValue()) + Result += arrayIdx * getTypeAllocSize(GTI.getIndexedType()); } } @@ -764,7 +758,7 @@ uint64_t DataLayout::getIndexedOffset(Type *ptrTy, /// global. This includes an explicitly requested alignment (if the global /// has one). unsigned DataLayout::getPreferredAlignment(const GlobalVariable *GV) const { - Type *ElemType = GV->getType()->getElementType(); + Type *ElemType = GV->getValueType(); unsigned Alignment = getPrefTypeAlignment(ElemType); unsigned GVAlignment = GV->getAlignment(); if (GVAlignment >= Alignment) { diff --git a/lib/IR/DebugInfo.cpp b/lib/IR/DebugInfo.cpp index a2443becdd00d..1d3c8299255b7 100644 --- a/lib/IR/DebugInfo.cpp +++ b/lib/IR/DebugInfo.cpp @@ -16,15 +16,13 @@ #include "LLVMContextImpl.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallString.h" -#include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DIBuilder.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GVMaterializer.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" -#include "llvm/IR/GVMaterializer.h" #include "llvm/IR/Module.h" #include "llvm/IR/ValueHandle.h" #include "llvm/Support/Debug.h" @@ -39,48 +37,6 @@ DISubprogram *llvm::getDISubprogram(const MDNode *Scope) { return nullptr; } -DISubprogram *llvm::getDISubprogram(const Function *F) { - // We look for the first instr that has a debug annotation leading back to F. - for (auto &BB : *F) { - auto Inst = std::find_if(BB.begin(), BB.end(), [](const Instruction &Inst) { - return Inst.getDebugLoc(); - }); - if (Inst == BB.end()) - continue; - DebugLoc DLoc = Inst->getDebugLoc(); - const MDNode *Scope = DLoc.getInlinedAtScope(); - auto *Subprogram = getDISubprogram(Scope); - return Subprogram->describes(F) ? Subprogram : nullptr; - } - - return nullptr; -} - -DITypeIdentifierMap -llvm::generateDITypeIdentifierMap(const NamedMDNode *CU_Nodes) { - DITypeIdentifierMap Map; - for (unsigned CUi = 0, CUe = CU_Nodes->getNumOperands(); CUi != CUe; ++CUi) { - auto *CU = cast<DICompileUnit>(CU_Nodes->getOperand(CUi)); - DINodeArray Retain = CU->getRetainedTypes(); - for (unsigned Ti = 0, Te = Retain.size(); Ti != Te; ++Ti) { - if (!isa<DICompositeType>(Retain[Ti])) - continue; - auto *Ty = cast<DICompositeType>(Retain[Ti]); - if (MDString *TypeId = Ty->getRawIdentifier()) { - // Definition has priority over declaration. - // Try to insert (TypeId, Ty) to Map. - std::pair<DITypeIdentifierMap::iterator, bool> P = - Map.insert(std::make_pair(TypeId, Ty)); - // If TypeId already exists in Map and this is a definition, replace - // whatever we had (declaration or definition) with the definition. - if (!P.second && !Ty->isForwardDecl()) - P.first->second = Ty; - } - } - } - return Map; -} - //===----------------------------------------------------------------------===// // DebugInfoFinder implementations. //===----------------------------------------------------------------------===// @@ -92,55 +48,44 @@ void DebugInfoFinder::reset() { TYs.clear(); Scopes.clear(); NodesSeen.clear(); - TypeIdentifierMap.clear(); - TypeMapInitialized = false; -} - -void DebugInfoFinder::InitializeTypeMap(const Module &M) { - if (!TypeMapInitialized) - if (NamedMDNode *CU_Nodes = M.getNamedMetadata("llvm.dbg.cu")) { - TypeIdentifierMap = generateDITypeIdentifierMap(CU_Nodes); - TypeMapInitialized = true; - } } void DebugInfoFinder::processModule(const Module &M) { - InitializeTypeMap(M); - if (NamedMDNode *CU_Nodes = M.getNamedMetadata("llvm.dbg.cu")) { - for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) { - auto *CU = cast<DICompileUnit>(CU_Nodes->getOperand(i)); - addCompileUnit(CU); - for (auto *DIG : CU->getGlobalVariables()) { - if (addGlobalVariable(DIG)) { - processScope(DIG->getScope()); - processType(DIG->getType().resolve(TypeIdentifierMap)); - } + for (auto *CU : M.debug_compile_units()) { + addCompileUnit(CU); + for (auto *DIG : CU->getGlobalVariables()) { + if (addGlobalVariable(DIG)) { + processScope(DIG->getScope()); + processType(DIG->getType().resolve()); } - for (auto *SP : CU->getSubprograms()) + } + for (auto *ET : CU->getEnumTypes()) + processType(ET); + for (auto *RT : CU->getRetainedTypes()) + if (auto *T = dyn_cast<DIType>(RT)) + processType(T); + else + processSubprogram(cast<DISubprogram>(RT)); + for (auto *Import : CU->getImportedEntities()) { + auto *Entity = Import->getEntity().resolve(); + if (auto *T = dyn_cast<DIType>(Entity)) + processType(T); + else if (auto *SP = dyn_cast<DISubprogram>(Entity)) processSubprogram(SP); - for (auto *ET : CU->getEnumTypes()) - processType(ET); - for (auto *RT : CU->getRetainedTypes()) - processType(RT); - for (auto *Import : CU->getImportedEntities()) { - auto *Entity = Import->getEntity().resolve(TypeIdentifierMap); - if (auto *T = dyn_cast<DIType>(Entity)) - processType(T); - else if (auto *SP = dyn_cast<DISubprogram>(Entity)) - processSubprogram(SP); - else if (auto *NS = dyn_cast<DINamespace>(Entity)) - processScope(NS->getScope()); - else if (auto *M = dyn_cast<DIModule>(Entity)) - processScope(M->getScope()); - } + else if (auto *NS = dyn_cast<DINamespace>(Entity)) + processScope(NS->getScope()); + else if (auto *M = dyn_cast<DIModule>(Entity)) + processScope(M->getScope()); } } + for (auto &F : M.functions()) + if (auto *SP = cast_or_null<DISubprogram>(F.getSubprogram())) + processSubprogram(SP); } void DebugInfoFinder::processLocation(const Module &M, const DILocation *Loc) { if (!Loc) return; - InitializeTypeMap(M); processScope(Loc->getScope()); processLocation(M, Loc->getInlinedAt()); } @@ -148,14 +93,14 @@ void DebugInfoFinder::processLocation(const Module &M, const DILocation *Loc) { void DebugInfoFinder::processType(DIType *DT) { if (!addType(DT)) return; - processScope(DT->getScope().resolve(TypeIdentifierMap)); + processScope(DT->getScope().resolve()); if (auto *ST = dyn_cast<DISubroutineType>(DT)) { for (DITypeRef Ref : ST->getTypeArray()) - processType(Ref.resolve(TypeIdentifierMap)); + processType(Ref.resolve()); return; } if (auto *DCT = dyn_cast<DICompositeType>(DT)) { - processType(DCT->getBaseType().resolve(TypeIdentifierMap)); + processType(DCT->getBaseType().resolve()); for (Metadata *D : DCT->getElements()) { if (auto *T = dyn_cast<DIType>(D)) processType(T); @@ -165,7 +110,7 @@ void DebugInfoFinder::processType(DIType *DT) { return; } if (auto *DDT = dyn_cast<DIDerivedType>(DT)) { - processType(DDT->getBaseType().resolve(TypeIdentifierMap)); + processType(DDT->getBaseType().resolve()); } } @@ -198,13 +143,13 @@ void DebugInfoFinder::processScope(DIScope *Scope) { void DebugInfoFinder::processSubprogram(DISubprogram *SP) { if (!addSubprogram(SP)) return; - processScope(SP->getScope().resolve(TypeIdentifierMap)); + processScope(SP->getScope().resolve()); processType(SP->getType()); for (auto *Element : SP->getTemplateParams()) { if (auto *TType = dyn_cast<DITemplateTypeParameter>(Element)) { - processType(TType->getType().resolve(TypeIdentifierMap)); + processType(TType->getType().resolve()); } else if (auto *TVal = dyn_cast<DITemplateValueParameter>(Element)) { - processType(TVal->getType().resolve(TypeIdentifierMap)); + processType(TVal->getType().resolve()); } } } @@ -214,7 +159,6 @@ void DebugInfoFinder::processDeclare(const Module &M, auto *N = dyn_cast<MDNode>(DDI->getVariable()); if (!N) return; - InitializeTypeMap(M); auto *DV = dyn_cast<DILocalVariable>(N); if (!DV) @@ -223,14 +167,13 @@ void DebugInfoFinder::processDeclare(const Module &M, if (!NodesSeen.insert(DV).second) return; processScope(DV->getScope()); - processType(DV->getType().resolve(TypeIdentifierMap)); + processType(DV->getType().resolve()); } void DebugInfoFinder::processValue(const Module &M, const DbgValueInst *DVI) { auto *N = dyn_cast<MDNode>(DVI->getVariable()); if (!N) return; - InitializeTypeMap(M); auto *DV = dyn_cast<DILocalVariable>(N); if (!DV) @@ -239,7 +182,7 @@ void DebugInfoFinder::processValue(const Module &M, const DbgValueInst *DVI) { if (!NodesSeen.insert(DV).second) return; processScope(DV->getScope()); - processType(DV->getType().resolve(TypeIdentifierMap)); + processType(DV->getType().resolve()); } bool DebugInfoFinder::addType(DIType *DT) { @@ -304,8 +247,15 @@ bool llvm::stripDebugInfo(Function &F) { Changed = true; F.setSubprogram(nullptr); } + for (BasicBlock &BB : F) { - for (Instruction &I : BB) { + for (auto II = BB.begin(), End = BB.end(); II != End;) { + Instruction &I = *II++; // We may delete the instruction, increment now. + if (isa<DbgInfoIntrinsic>(&I)) { + I.eraseFromParent(); + Changed = true; + continue; + } if (I.getDebugLoc()) { Changed = true; I.setDebugLoc(DebugLoc()); @@ -318,26 +268,6 @@ bool llvm::stripDebugInfo(Function &F) { bool llvm::StripDebugInfo(Module &M) { bool Changed = false; - // Remove all of the calls to the debugger intrinsics, and remove them from - // the module. - if (Function *Declare = M.getFunction("llvm.dbg.declare")) { - while (!Declare->use_empty()) { - CallInst *CI = cast<CallInst>(Declare->user_back()); - CI->eraseFromParent(); - } - Declare->eraseFromParent(); - Changed = true; - } - - if (Function *DbgVal = M.getFunction("llvm.dbg.value")) { - while (!DbgVal->use_empty()) { - CallInst *CI = cast<CallInst>(DbgVal->user_back()); - CI->eraseFromParent(); - } - DbgVal->eraseFromParent(); - Changed = true; - } - for (Module::named_metadata_iterator NMI = M.named_metadata_begin(), NME = M.named_metadata_end(); NMI != NME;) { NamedMDNode *NMD = &*NMI; diff --git a/lib/IR/DebugInfoMetadata.cpp b/lib/IR/DebugInfoMetadata.cpp index 58e0abdd577c0..c58e3685b3c9e 100644 --- a/lib/IR/DebugInfoMetadata.cpp +++ b/lib/IR/DebugInfoMetadata.cpp @@ -45,7 +45,6 @@ DILocation *DILocation::getImpl(LLVMContext &Context, unsigned Line, // Fixup column. adjustColumn(Column); - assert(Scope && "Expected scope"); if (Storage == Uniqued) { if (auto *N = getUniqued(Context.pImpl->DILocations, @@ -66,23 +65,6 @@ DILocation *DILocation::getImpl(LLVMContext &Context, unsigned Line, Storage, Context.pImpl->DILocations); } -unsigned DILocation::computeNewDiscriminator() const { - // FIXME: This seems completely wrong. - // - // 1. If two modules are generated in the same context, then the second - // Module will get different discriminators than it would have if it were - // generated in its own context. - // 2. If this function is called after round-tripping to bitcode instead of - // before, it will give a different (and potentially incorrect!) return. - // - // The discriminator should instead be calculated from local information - // where it's actually needed. This logic should be moved to - // AddDiscriminators::runOnFunction(), where it doesn't pollute the - // LLVMContext. - std::pair<const char *, unsigned> Key(getFilename().data(), getLine()); - return ++getContext().pImpl->DiscriminatorTable[Key]; -} - unsigned DINode::getFlag(StringRef Flag) { return StringSwitch<unsigned>(Flag) #define HANDLE_DI_FLAG(ID, NAME) .Case("DIFlag" #NAME, Flag##NAME) @@ -103,8 +85,8 @@ const char *DINode::getFlagString(unsigned Flag) { unsigned DINode::splitFlags(unsigned Flags, SmallVectorImpl<unsigned> &SplitFlags) { - // Accessibility flags need to be specially handled, since they're packed - // together. + // Accessibility and member pointer flags need to be specially handled, since + // they're packed together. if (unsigned A = Flags & FlagAccessibility) { if (A == FlagPrivate) SplitFlags.push_back(FlagPrivate); @@ -114,6 +96,15 @@ unsigned DINode::splitFlags(unsigned Flags, SplitFlags.push_back(FlagPublic); Flags &= ~A; } + if (unsigned R = Flags & FlagPtrToMemberRep) { + if (R == FlagSingleInheritance) + SplitFlags.push_back(FlagSingleInheritance); + else if (R == FlagMultipleInheritance) + SplitFlags.push_back(FlagMultipleInheritance); + else + SplitFlags.push_back(FlagVirtualInheritance); + Flags &= ~R; + } #define HANDLE_DI_FLAG(ID, NAME) \ if (unsigned Bit = Flags & ID) { \ @@ -133,13 +124,13 @@ DIScopeRef DIScope::getScope() const { return SP->getScope(); if (auto *LB = dyn_cast<DILexicalBlockBase>(this)) - return DIScopeRef(LB->getScope()); + return LB->getScope(); if (auto *NS = dyn_cast<DINamespace>(this)) - return DIScopeRef(NS->getScope()); + return NS->getScope(); if (auto *M = dyn_cast<DIModule>(this)) - return DIScopeRef(M->getScope()); + return M->getScope(); assert((isa<DIFile>(this) || isa<DICompileUnit>(this)) && "Unhandled type of scope."); @@ -161,12 +152,6 @@ StringRef DIScope::getName() const { return ""; } -static StringRef getString(const MDString *S) { - if (S) - return S->getString(); - return StringRef(); -} - #ifndef NDEBUG static bool isCanonical(const MDString *S) { return !S || !S->getString().empty(); @@ -179,7 +164,7 @@ GenericDINode *GenericDINode::getImpl(LLVMContext &Context, unsigned Tag, StorageType Storage, bool ShouldCreate) { unsigned Hash = 0; if (Storage == Uniqued) { - GenericDINodeInfo::KeyTy Key(Tag, getString(Header), DwarfOps); + GenericDINodeInfo::KeyTy Key(Tag, Header, DwarfOps); if (auto *N = getUniqued(Context.pImpl->GenericDINodes, Key)) return N; if (!ShouldCreate) @@ -217,15 +202,14 @@ void GenericDINode::recalculateHash() { } \ } while (false) #define DEFINE_GETIMPL_STORE(CLASS, ARGS, OPS) \ - return storeImpl(new (ArrayRef<Metadata *>(OPS).size()) \ + return storeImpl(new (array_lengthof(OPS)) \ CLASS(Context, Storage, UNWRAP_ARGS(ARGS), OPS), \ Storage, Context.pImpl->CLASS##s) #define DEFINE_GETIMPL_STORE_NO_OPS(CLASS, ARGS) \ return storeImpl(new (0u) CLASS(Context, Storage, UNWRAP_ARGS(ARGS)), \ Storage, Context.pImpl->CLASS##s) #define DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(CLASS, OPS) \ - return storeImpl(new (ArrayRef<Metadata *>(OPS).size()) \ - CLASS(Context, Storage, OPS), \ + return storeImpl(new (array_lengthof(OPS)) CLASS(Context, Storage, OPS), \ Storage, Context.pImpl->CLASS##s) DISubrange *DISubrange::getImpl(LLVMContext &Context, int64_t Count, int64_t Lo, @@ -238,7 +222,7 @@ DIEnumerator *DIEnumerator::getImpl(LLVMContext &Context, int64_t Value, MDString *Name, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DIEnumerator, (Value, getString(Name))); + DEFINE_GETIMPL_LOOKUP(DIEnumerator, (Value, Name)); Metadata *Ops[] = {Name}; DEFINE_GETIMPL_STORE(DIEnumerator, (Value), Ops); } @@ -248,8 +232,8 @@ DIBasicType *DIBasicType::getImpl(LLVMContext &Context, unsigned Tag, uint64_t AlignInBits, unsigned Encoding, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP( - DIBasicType, (Tag, getString(Name), SizeInBits, AlignInBits, Encoding)); + DEFINE_GETIMPL_LOOKUP(DIBasicType, + (Tag, Name, SizeInBits, AlignInBits, Encoding)); Metadata *Ops[] = {nullptr, nullptr, Name}; DEFINE_GETIMPL_STORE(DIBasicType, (Tag, SizeInBits, AlignInBits, Encoding), Ops); @@ -261,9 +245,9 @@ DIDerivedType *DIDerivedType::getImpl( uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, Metadata *ExtraData, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DIDerivedType, (Tag, getString(Name), File, Line, Scope, - BaseType, SizeInBits, AlignInBits, - OffsetInBits, Flags, ExtraData)); + DEFINE_GETIMPL_LOOKUP(DIDerivedType, + (Tag, Name, File, Line, Scope, BaseType, SizeInBits, + AlignInBits, OffsetInBits, Flags, ExtraData)); Metadata *Ops[] = {File, Scope, Name, BaseType, ExtraData}; DEFINE_GETIMPL_STORE( DIDerivedType, (Tag, Line, SizeInBits, AlignInBits, OffsetInBits, Flags), @@ -278,11 +262,12 @@ DICompositeType *DICompositeType::getImpl( Metadata *TemplateParams, MDString *Identifier, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DICompositeType, - (Tag, getString(Name), File, Line, Scope, BaseType, - SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, - RuntimeLang, VTableHolder, TemplateParams, - getString(Identifier))); + + // Keep this in sync with buildODRType. + DEFINE_GETIMPL_LOOKUP( + DICompositeType, (Tag, Name, File, Line, Scope, BaseType, SizeInBits, + AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, + VTableHolder, TemplateParams, Identifier)); Metadata *Ops[] = {File, Scope, Name, BaseType, Elements, VTableHolder, TemplateParams, Identifier}; DEFINE_GETIMPL_STORE(DICompositeType, (Tag, Line, RuntimeLang, SizeInBits, @@ -290,13 +275,74 @@ DICompositeType *DICompositeType::getImpl( Ops); } +DICompositeType *DICompositeType::buildODRType( + LLVMContext &Context, MDString &Identifier, unsigned Tag, MDString *Name, + Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, + uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, + unsigned Flags, Metadata *Elements, unsigned RuntimeLang, + Metadata *VTableHolder, Metadata *TemplateParams) { + assert(!Identifier.getString().empty() && "Expected valid identifier"); + if (!Context.isODRUniquingDebugTypes()) + return nullptr; + auto *&CT = (*Context.pImpl->DITypeMap)[&Identifier]; + if (!CT) + return CT = DICompositeType::getDistinct( + Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, + AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, + VTableHolder, TemplateParams, &Identifier); + + // Only mutate CT if it's a forward declaration and the new operands aren't. + assert(CT->getRawIdentifier() == &Identifier && "Wrong ODR identifier?"); + if (!CT->isForwardDecl() || (Flags & DINode::FlagFwdDecl)) + return CT; + + // Mutate CT in place. Keep this in sync with getImpl. + CT->mutate(Tag, Line, RuntimeLang, SizeInBits, AlignInBits, OffsetInBits, + Flags); + Metadata *Ops[] = {File, Scope, Name, BaseType, + Elements, VTableHolder, TemplateParams, &Identifier}; + assert((std::end(Ops) - std::begin(Ops)) == (int)CT->getNumOperands() && + "Mismatched number of operands"); + for (unsigned I = 0, E = CT->getNumOperands(); I != E; ++I) + if (Ops[I] != CT->getOperand(I)) + CT->setOperand(I, Ops[I]); + return CT; +} + +DICompositeType *DICompositeType::getODRType( + LLVMContext &Context, MDString &Identifier, unsigned Tag, MDString *Name, + Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, + uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, + unsigned Flags, Metadata *Elements, unsigned RuntimeLang, + Metadata *VTableHolder, Metadata *TemplateParams) { + assert(!Identifier.getString().empty() && "Expected valid identifier"); + if (!Context.isODRUniquingDebugTypes()) + return nullptr; + auto *&CT = (*Context.pImpl->DITypeMap)[&Identifier]; + if (!CT) + CT = DICompositeType::getDistinct( + Context, Tag, Name, File, Line, Scope, BaseType, SizeInBits, + AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, + TemplateParams, &Identifier); + return CT; +} + +DICompositeType *DICompositeType::getODRTypeIfExists(LLVMContext &Context, + MDString &Identifier) { + assert(!Identifier.getString().empty() && "Expected valid identifier"); + if (!Context.isODRUniquingDebugTypes()) + return nullptr; + return Context.pImpl->DITypeMap->lookup(&Identifier); +} + DISubroutineType *DISubroutineType::getImpl(LLVMContext &Context, - unsigned Flags, Metadata *TypeArray, + unsigned Flags, uint8_t CC, + Metadata *TypeArray, StorageType Storage, bool ShouldCreate) { - DEFINE_GETIMPL_LOOKUP(DISubroutineType, (Flags, TypeArray)); + DEFINE_GETIMPL_LOOKUP(DISubroutineType, (Flags, CC, TypeArray)); Metadata *Ops[] = {nullptr, nullptr, nullptr, TypeArray}; - DEFINE_GETIMPL_STORE(DISubroutineType, (Flags), Ops); + DEFINE_GETIMPL_STORE(DISubroutineType, (Flags, CC), Ops); } DIFile *DIFile::getImpl(LLVMContext &Context, MDString *Filename, @@ -304,7 +350,7 @@ DIFile *DIFile::getImpl(LLVMContext &Context, MDString *Filename, bool ShouldCreate) { assert(isCanonical(Filename) && "Expected canonical MDString"); assert(isCanonical(Directory) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DIFile, (getString(Filename), getString(Directory))); + DEFINE_GETIMPL_LOOKUP(DIFile, (Filename, Directory)); Metadata *Ops[] = {Filename, Directory}; DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DIFile, Ops); } @@ -314,50 +360,74 @@ DICompileUnit *DICompileUnit::getImpl( MDString *Producer, bool IsOptimized, MDString *Flags, unsigned RuntimeVersion, MDString *SplitDebugFilename, unsigned EmissionKind, Metadata *EnumTypes, Metadata *RetainedTypes, - Metadata *Subprograms, Metadata *GlobalVariables, - Metadata *ImportedEntities, Metadata *Macros, uint64_t DWOId, - StorageType Storage, bool ShouldCreate) { + Metadata *GlobalVariables, Metadata *ImportedEntities, Metadata *Macros, + uint64_t DWOId, StorageType Storage, bool ShouldCreate) { assert(Storage != Uniqued && "Cannot unique DICompileUnit"); assert(isCanonical(Producer) && "Expected canonical MDString"); assert(isCanonical(Flags) && "Expected canonical MDString"); assert(isCanonical(SplitDebugFilename) && "Expected canonical MDString"); - Metadata *Ops[] = {File, Producer, Flags, SplitDebugFilename, EnumTypes, - RetainedTypes, Subprograms, GlobalVariables, - ImportedEntities, Macros}; - return storeImpl(new (ArrayRef<Metadata *>(Ops).size()) DICompileUnit( + Metadata *Ops[] = { + File, Producer, Flags, SplitDebugFilename, + EnumTypes, RetainedTypes, GlobalVariables, ImportedEntities, + Macros}; + return storeImpl(new (array_lengthof(Ops)) DICompileUnit( Context, Storage, SourceLanguage, IsOptimized, RuntimeVersion, EmissionKind, DWOId, Ops), Storage); } +Optional<DICompileUnit::DebugEmissionKind> +DICompileUnit::getEmissionKind(StringRef Str) { + return StringSwitch<Optional<DebugEmissionKind>>(Str) + .Case("NoDebug", NoDebug) + .Case("FullDebug", FullDebug) + .Case("LineTablesOnly", LineTablesOnly) + .Default(None); +} + +const char *DICompileUnit::EmissionKindString(DebugEmissionKind EK) { + switch (EK) { + case NoDebug: return "NoDebug"; + case FullDebug: return "FullDebug"; + case LineTablesOnly: return "LineTablesOnly"; + } + return nullptr; +} + DISubprogram *DILocalScope::getSubprogram() const { if (auto *Block = dyn_cast<DILexicalBlockBase>(this)) return Block->getScope()->getSubprogram(); return const_cast<DISubprogram *>(cast<DISubprogram>(this)); } +DILocalScope *DILocalScope::getNonLexicalBlockFileScope() const { + if (auto *File = dyn_cast<DILexicalBlockFile>(this)) + return File->getScope()->getNonLexicalBlockFileScope(); + return const_cast<DILocalScope *>(this); +} + DISubprogram *DISubprogram::getImpl( LLVMContext &Context, Metadata *Scope, MDString *Name, MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, Metadata *ContainingType, unsigned Virtuality, unsigned VirtualIndex, - unsigned Flags, bool IsOptimized, Metadata *TemplateParams, - Metadata *Declaration, Metadata *Variables, StorageType Storage, - bool ShouldCreate) { + int ThisAdjustment, unsigned Flags, bool IsOptimized, Metadata *Unit, + Metadata *TemplateParams, Metadata *Declaration, Metadata *Variables, + StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); assert(isCanonical(LinkageName) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DISubprogram, - (Scope, getString(Name), getString(LinkageName), File, - Line, Type, IsLocalToUnit, IsDefinition, ScopeLine, - ContainingType, Virtuality, VirtualIndex, Flags, - IsOptimized, TemplateParams, Declaration, Variables)); - Metadata *Ops[] = {File, Scope, Name, Name, - LinkageName, Type, ContainingType, TemplateParams, - Declaration, Variables}; - DEFINE_GETIMPL_STORE(DISubprogram, - (Line, ScopeLine, Virtuality, VirtualIndex, Flags, - IsLocalToUnit, IsDefinition, IsOptimized), + DEFINE_GETIMPL_LOOKUP( + DISubprogram, + (Scope, Name, LinkageName, File, Line, Type, IsLocalToUnit, IsDefinition, + ScopeLine, ContainingType, Virtuality, VirtualIndex, ThisAdjustment, + Flags, IsOptimized, Unit, TemplateParams, Declaration, Variables)); + Metadata *Ops[] = {File, Scope, Name, Name, + LinkageName, Type, ContainingType, Unit, + TemplateParams, Declaration, Variables}; + DEFINE_GETIMPL_STORE(DISubprogram, (Line, ScopeLine, Virtuality, VirtualIndex, + ThisAdjustment, Flags, IsLocalToUnit, + IsDefinition, IsOptimized), Ops); } @@ -399,7 +469,7 @@ DINamespace *DINamespace::getImpl(LLVMContext &Context, Metadata *Scope, Metadata *File, MDString *Name, unsigned Line, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DINamespace, (Scope, File, getString(Name), Line)); + DEFINE_GETIMPL_LOOKUP(DINamespace, (Scope, File, Name, Line)); Metadata *Ops[] = {File, Scope, Name}; DEFINE_GETIMPL_STORE(DINamespace, (Line), Ops); } @@ -409,9 +479,8 @@ DIModule *DIModule::getImpl(LLVMContext &Context, Metadata *Scope, MDString *IncludePath, MDString *ISysRoot, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DIModule, - (Scope, getString(Name), getString(ConfigurationMacros), - getString(IncludePath), getString(ISysRoot))); + DEFINE_GETIMPL_LOOKUP( + DIModule, (Scope, Name, ConfigurationMacros, IncludePath, ISysRoot)); Metadata *Ops[] = {Scope, Name, ConfigurationMacros, IncludePath, ISysRoot}; DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DIModule, Ops); } @@ -422,7 +491,7 @@ DITemplateTypeParameter *DITemplateTypeParameter::getImpl(LLVMContext &Context, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DITemplateTypeParameter, (getString(Name), Type)); + DEFINE_GETIMPL_LOOKUP(DITemplateTypeParameter, (Name, Type)); Metadata *Ops[] = {Name, Type}; DEFINE_GETIMPL_STORE_NO_CONSTRUCTOR_ARGS(DITemplateTypeParameter, Ops); } @@ -431,8 +500,7 @@ DITemplateValueParameter *DITemplateValueParameter::getImpl( LLVMContext &Context, unsigned Tag, MDString *Name, Metadata *Type, Metadata *Value, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DITemplateValueParameter, - (Tag, getString(Name), Type, Value)); + DEFINE_GETIMPL_LOOKUP(DITemplateValueParameter, (Tag, Name, Type, Value)); Metadata *Ops[] = {Name, Type, Value}; DEFINE_GETIMPL_STORE(DITemplateValueParameter, (Tag), Ops); } @@ -447,8 +515,8 @@ DIGlobalVariable::getImpl(LLVMContext &Context, Metadata *Scope, MDString *Name, assert(isCanonical(Name) && "Expected canonical MDString"); assert(isCanonical(LinkageName) && "Expected canonical MDString"); DEFINE_GETIMPL_LOOKUP(DIGlobalVariable, - (Scope, getString(Name), getString(LinkageName), File, - Line, Type, IsLocalToUnit, IsDefinition, Variable, + (Scope, Name, LinkageName, File, Line, Type, + IsLocalToUnit, IsDefinition, Variable, StaticDataMemberDeclaration)); Metadata *Ops[] = {Scope, Name, File, Type, Name, LinkageName, Variable, StaticDataMemberDeclaration}; @@ -468,7 +536,7 @@ DILocalVariable *DILocalVariable::getImpl(LLVMContext &Context, Metadata *Scope, assert(Scope && "Expected scope"); assert(isCanonical(Name) && "Expected canonical MDString"); DEFINE_GETIMPL_LOOKUP(DILocalVariable, - (Scope, getString(Name), File, Line, Type, Arg, Flags)); + (Scope, Name, File, Line, Type, Arg, Flags)); Metadata *Ops[] = {Scope, Name, File, Type}; DEFINE_GETIMPL_STORE(DILocalVariable, (Line, Arg, Flags), Ops); } @@ -539,9 +607,8 @@ DIObjCProperty *DIObjCProperty::getImpl( assert(isCanonical(Name) && "Expected canonical MDString"); assert(isCanonical(GetterName) && "Expected canonical MDString"); assert(isCanonical(SetterName) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DIObjCProperty, - (getString(Name), File, Line, getString(GetterName), - getString(SetterName), Attributes, Type)); + DEFINE_GETIMPL_LOOKUP(DIObjCProperty, (Name, File, Line, GetterName, + SetterName, Attributes, Type)); Metadata *Ops[] = {Name, File, GetterName, SetterName, Type}; DEFINE_GETIMPL_STORE(DIObjCProperty, (Line, Attributes), Ops); } @@ -552,8 +619,7 @@ DIImportedEntity *DIImportedEntity::getImpl(LLVMContext &Context, unsigned Tag, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DIImportedEntity, - (Tag, Scope, Entity, Line, getString(Name))); + DEFINE_GETIMPL_LOOKUP(DIImportedEntity, (Tag, Scope, Entity, Line, Name)); Metadata *Ops[] = {Scope, Entity, Name}; DEFINE_GETIMPL_STORE(DIImportedEntity, (Tag, Line), Ops); } @@ -562,8 +628,7 @@ DIMacro *DIMacro::getImpl(LLVMContext &Context, unsigned MIType, unsigned Line, MDString *Name, MDString *Value, StorageType Storage, bool ShouldCreate) { assert(isCanonical(Name) && "Expected canonical MDString"); - DEFINE_GETIMPL_LOOKUP(DIMacro, - (MIType, Line, getString(Name), getString(Value))); + DEFINE_GETIMPL_LOOKUP(DIMacro, (MIType, Line, Name, Value)); Metadata *Ops[] = { Name, Value }; DEFINE_GETIMPL_STORE(DIMacro, (MIType, Line), Ops); } diff --git a/lib/IR/DebugLoc.cpp b/lib/IR/DebugLoc.cpp index 72d5c0e618833..ffa7a6b40e2a6 100644 --- a/lib/IR/DebugLoc.cpp +++ b/lib/IR/DebugLoc.cpp @@ -9,7 +9,6 @@ #include "llvm/IR/DebugLoc.h" #include "LLVMContextImpl.h" -#include "llvm/ADT/DenseMapInfo.h" #include "llvm/IR/DebugInfo.h" using namespace llvm; @@ -67,7 +66,7 @@ DebugLoc DebugLoc::get(unsigned Line, unsigned Col, const MDNode *Scope, const_cast<MDNode *>(InlinedAt)); } -void DebugLoc::dump() const { +LLVM_DUMP_METHOD void DebugLoc::dump() const { #ifndef NDEBUG if (!Loc) return; diff --git a/lib/IR/DiagnosticInfo.cpp b/lib/IR/DiagnosticInfo.cpp index 6426f76bbaa61..ce67be328ab17 100644 --- a/lib/IR/DiagnosticInfo.cpp +++ b/lib/IR/DiagnosticInfo.cpp @@ -91,7 +91,7 @@ int llvm::getNextAvailablePluginDiagnosticKind() { return ++PluginKindID; } -const char *DiagnosticInfo::AlwaysPrint = ""; +const char *DiagnosticInfoOptimizationRemarkAnalysis::AlwaysPrint = ""; DiagnosticInfoInlineAsm::DiagnosticInfoInlineAsm(const Instruction &I, const Twine &MsgStr, @@ -112,9 +112,13 @@ void DiagnosticInfoInlineAsm::print(DiagnosticPrinter &DP) const { DP << " at line " << getLocCookie(); } -void DiagnosticInfoStackSize::print(DiagnosticPrinter &DP) const { - DP << "stack size limit exceeded (" << getStackSize() << ") in " - << getFunction(); +void DiagnosticInfoResourceLimit::print(DiagnosticPrinter &DP) const { + DP << getResourceName() << " limit"; + + if (getResourceLimit() != 0) + DP << " of " << getResourceLimit(); + + DP << " exceeded (" << getResourceSize() << ") in " << getFunction(); } void DiagnosticInfoDebugMetadataVersion::print(DiagnosticPrinter &DP) const { @@ -122,6 +126,11 @@ void DiagnosticInfoDebugMetadataVersion::print(DiagnosticPrinter &DP) const { << ") in " << getModule(); } +void DiagnosticInfoIgnoringInvalidDebugMetadata::print( + DiagnosticPrinter &DP) const { + DP << "ignoring invalid debug info in " << getModule().getModuleIdentifier(); +} + void DiagnosticInfoSampleProfile::print(DiagnosticPrinter &DP) const { if (!FileName.empty()) { DP << getFileName(); @@ -138,11 +147,11 @@ void DiagnosticInfoPGOProfile::print(DiagnosticPrinter &DP) const { DP << getMsg(); } -bool DiagnosticInfoOptimizationBase::isLocationAvailable() const { +bool DiagnosticInfoWithDebugLocBase::isLocationAvailable() const { return getDebugLoc(); } -void DiagnosticInfoOptimizationBase::getLocation(StringRef *Filename, +void DiagnosticInfoWithDebugLocBase::getLocation(StringRef *Filename, unsigned *Line, unsigned *Column) const { DILocation *L = getDebugLoc(); @@ -152,7 +161,7 @@ void DiagnosticInfoOptimizationBase::getLocation(StringRef *Filename, *Column = L->getColumn(); } -const std::string DiagnosticInfoOptimizationBase::getLocationStr() const { +const std::string DiagnosticInfoWithDebugLocBase::getLocationStr() const { StringRef Filename("<unknown>"); unsigned Line = 0; unsigned Column = 0; @@ -163,6 +172,8 @@ const std::string DiagnosticInfoOptimizationBase::getLocationStr() const { void DiagnosticInfoOptimizationBase::print(DiagnosticPrinter &DP) const { DP << getLocationStr() << ": " << getMsg(); + if (Hotness) + DP << " (hotness: " << *Hotness << ")"; } bool DiagnosticInfoOptimizationRemark::isEnabled() const { @@ -176,7 +187,7 @@ bool DiagnosticInfoOptimizationRemarkMissed::isEnabled() const { } bool DiagnosticInfoOptimizationRemarkAnalysis::isEnabled() const { - return getPassName() == DiagnosticInfo::AlwaysPrint || + return shouldAlwaysPrint() || (PassRemarksAnalysisOptLoc.Pattern && PassRemarksAnalysisOptLoc.Pattern->match(getPassName())); } @@ -230,6 +241,16 @@ bool DiagnosticInfoOptimizationFailure::isEnabled() const { return getSeverity() == DS_Warning; } +void DiagnosticInfoUnsupported::print(DiagnosticPrinter &DP) const { + std::string Str; + raw_string_ostream OS(Str); + + OS << getLocationStr() << ": in function " << getFunction().getName() << ' ' + << *getFunction().getFunctionType() << ": " << Msg << '\n'; + OS.flush(); + DP << Str; +} + void llvm::emitLoopVectorizeWarning(LLVMContext &Ctx, const Function &Fn, const DebugLoc &DLoc, const Twine &Msg) { Ctx.diagnose(DiagnosticInfoOptimizationFailure( diff --git a/lib/IR/Dominators.cpp b/lib/IR/Dominators.cpp index b9d4fb7de8815..57e3df76d0231 100644 --- a/lib/IR/Dominators.cpp +++ b/lib/IR/Dominators.cpp @@ -17,12 +17,10 @@ #include "llvm/IR/Dominators.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" #include "llvm/Support/CommandLine.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/GenericDomTreeConstruction.h" #include "llvm/Support/raw_ostream.h" @@ -30,7 +28,7 @@ using namespace llvm; // Always verify dominfo if expensive checking is enabled. -#ifdef XDEBUG +#ifdef EXPENSIVE_CHECKS static bool VerifyDomInfo = true; #else static bool VerifyDomInfo = false; @@ -302,7 +300,8 @@ void DominatorTree::verifyDomTree() const { // //===----------------------------------------------------------------------===// -DominatorTree DominatorTreeAnalysis::run(Function &F) { +DominatorTree DominatorTreeAnalysis::run(Function &F, + AnalysisManager<Function> &) { DominatorTree DT; DT.recalculate(F); return DT; @@ -313,16 +312,16 @@ char DominatorTreeAnalysis::PassID; DominatorTreePrinterPass::DominatorTreePrinterPass(raw_ostream &OS) : OS(OS) {} PreservedAnalyses DominatorTreePrinterPass::run(Function &F, - FunctionAnalysisManager *AM) { + FunctionAnalysisManager &AM) { OS << "DominatorTree for function: " << F.getName() << "\n"; - AM->getResult<DominatorTreeAnalysis>(F).print(OS); + AM.getResult<DominatorTreeAnalysis>(F).print(OS); return PreservedAnalyses::all(); } PreservedAnalyses DominatorTreeVerifierPass::run(Function &F, - FunctionAnalysisManager *AM) { - AM->getResult<DominatorTreeAnalysis>(F).verifyDomTree(); + FunctionAnalysisManager &AM) { + AM.getResult<DominatorTreeAnalysis>(F).verifyDomTree(); return PreservedAnalyses::all(); } diff --git a/lib/IR/Function.cpp b/lib/IR/Function.cpp index cfdfc40cd8aa7..e1223d0d03379 100644 --- a/lib/IR/Function.cpp +++ b/lib/IR/Function.cpp @@ -14,7 +14,6 @@ #include "llvm/IR/Function.h" #include "LLVMContextImpl.h" #include "SymbolTableListTraitsImpl.h" -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/CodeGen/ValueTypes.h" @@ -89,16 +88,24 @@ bool Argument::hasNonNullAttr() const { /// in its containing function. bool Argument::hasByValAttr() const { if (!getType()->isPointerTy()) return false; + return hasAttribute(Attribute::ByVal); +} + +bool Argument::hasSwiftSelfAttr() const { + return getParent()->getAttributes(). + hasAttribute(getArgNo()+1, Attribute::SwiftSelf); +} + +bool Argument::hasSwiftErrorAttr() const { return getParent()->getAttributes(). - hasAttribute(getArgNo()+1, Attribute::ByVal); + hasAttribute(getArgNo()+1, Attribute::SwiftError); } /// \brief Return true if this argument has the inalloca attribute on it in /// its containing function. bool Argument::hasInAllocaAttr() const { if (!getType()->isPointerTy()) return false; - return getParent()->getAttributes(). - hasAttribute(getArgNo()+1, Attribute::InAlloca); + return hasAttribute(Attribute::InAlloca); } bool Argument::hasByValOrInAllocaAttr() const { @@ -130,53 +137,46 @@ uint64_t Argument::getDereferenceableOrNullBytes() const { /// it in its containing function. bool Argument::hasNestAttr() const { if (!getType()->isPointerTy()) return false; - return getParent()->getAttributes(). - hasAttribute(getArgNo()+1, Attribute::Nest); + return hasAttribute(Attribute::Nest); } /// hasNoAliasAttr - Return true if this argument has the noalias attribute on /// it in its containing function. bool Argument::hasNoAliasAttr() const { if (!getType()->isPointerTy()) return false; - return getParent()->getAttributes(). - hasAttribute(getArgNo()+1, Attribute::NoAlias); + return hasAttribute(Attribute::NoAlias); } /// hasNoCaptureAttr - Return true if this argument has the nocapture attribute /// on it in its containing function. bool Argument::hasNoCaptureAttr() const { if (!getType()->isPointerTy()) return false; - return getParent()->getAttributes(). - hasAttribute(getArgNo()+1, Attribute::NoCapture); + return hasAttribute(Attribute::NoCapture); } /// hasSRetAttr - Return true if this argument has the sret attribute on /// it in its containing function. bool Argument::hasStructRetAttr() const { if (!getType()->isPointerTy()) return false; - return getParent()->getAttributes(). - hasAttribute(getArgNo()+1, Attribute::StructRet); + return hasAttribute(Attribute::StructRet); } /// hasReturnedAttr - Return true if this argument has the returned attribute on /// it in its containing function. bool Argument::hasReturnedAttr() const { - return getParent()->getAttributes(). - hasAttribute(getArgNo()+1, Attribute::Returned); + return hasAttribute(Attribute::Returned); } /// hasZExtAttr - Return true if this argument has the zext attribute on it in /// its containing function. bool Argument::hasZExtAttr() const { - return getParent()->getAttributes(). - hasAttribute(getArgNo()+1, Attribute::ZExt); + return hasAttribute(Attribute::ZExt); } /// hasSExtAttr Return true if this argument has the sext attribute on it in its /// containing function. bool Argument::hasSExtAttr() const { - return getParent()->getAttributes(). - hasAttribute(getArgNo()+1, Attribute::SExt); + return hasAttribute(Attribute::SExt); } /// Return true if this argument has the readonly or readnone attribute on it @@ -208,23 +208,32 @@ void Argument::removeAttr(AttributeSet AS) { getArgNo() + 1, B)); } +/// hasAttribute - Checks if an argument has a given attribute. +bool Argument::hasAttribute(Attribute::AttrKind Kind) const { + return getParent()->hasAttribute(getArgNo() + 1, Kind); +} + //===----------------------------------------------------------------------===// // Helper Methods in Function //===----------------------------------------------------------------------===// bool Function::isMaterializable() const { - return getGlobalObjectSubClassData() & IsMaterializableBit; + return getGlobalObjectSubClassData() & (1 << IsMaterializableBit); } void Function::setIsMaterializable(bool V) { - setGlobalObjectBit(IsMaterializableBit, V); + unsigned Mask = 1 << IsMaterializableBit; + setGlobalObjectSubClassData((~Mask & getGlobalObjectSubClassData()) | + (V ? Mask : 0u)); } LLVMContext &Function::getContext() const { return getType()->getContext(); } -FunctionType *Function::getFunctionType() const { return Ty; } +FunctionType *Function::getFunctionType() const { + return cast<FunctionType>(getValueType()); +} bool Function::isVarArg() const { return getFunctionType()->isVarArg(); @@ -249,8 +258,7 @@ void Function::eraseFromParent() { Function::Function(FunctionType *Ty, LinkageTypes Linkage, const Twine &name, Module *ParentModule) : GlobalObject(Ty, Value::FunctionVal, - OperandTraits<Function>::op_begin(this), 0, Linkage, name), - Ty(Ty) { + OperandTraits<Function>::op_begin(this), 0, Linkage, name) { assert(FunctionType::isValidReturnType(getReturnType()) && "invalid return type"); setGlobalObjectSubClassData(0); @@ -295,6 +303,28 @@ void Function::BuildLazyArguments() const { const_cast<Function*>(this)->setValueSubclassData(SDC &= ~(1<<0)); } +void Function::stealArgumentListFrom(Function &Src) { + assert(isDeclaration() && "Expected no references to current arguments"); + + // Drop the current arguments, if any, and set the lazy argument bit. + if (!hasLazyArguments()) { + assert(llvm::all_of(ArgumentList, + [](const Argument &A) { return A.use_empty(); }) && + "Expected arguments to be unused in declaration"); + ArgumentList.clear(); + setValueSubclassData(getSubclassDataFromValue() | (1 << 0)); + } + + // Nothing to steal if Src has lazy arguments. + if (Src.hasLazyArguments()) + return; + + // Steal arguments from Src, and fix the lazy argument bits. + ArgumentList.splice(ArgumentList.end(), Src.ArgumentList); + setValueSubclassData(getSubclassDataFromValue() & ~(1 << 0)); + Src.setValueSubclassData(Src.getSubclassDataFromValue() | (1 << 0)); +} + size_t Function::arg_size() const { return getFunctionType()->getNumParams(); } @@ -317,8 +347,8 @@ void Function::setParent(Module *parent) { void Function::dropAllReferences() { setIsMaterializable(false); - for (iterator I = begin(), E = end(); I != E; ++I) - I->dropAllReferences(); + for (BasicBlock &BB : *this) + BB.dropAllReferences(); // Delete all basic blocks. They are now unused, except possibly by // blockaddresses, but BasicBlock's destructor takes care of those. @@ -336,21 +366,39 @@ void Function::dropAllReferences() { clearMetadata(); } -void Function::addAttribute(unsigned i, Attribute::AttrKind attr) { +void Function::addAttribute(unsigned i, Attribute::AttrKind Kind) { AttributeSet PAL = getAttributes(); - PAL = PAL.addAttribute(getContext(), i, attr); + PAL = PAL.addAttribute(getContext(), i, Kind); setAttributes(PAL); } -void Function::addAttributes(unsigned i, AttributeSet attrs) { +void Function::addAttribute(unsigned i, Attribute Attr) { AttributeSet PAL = getAttributes(); - PAL = PAL.addAttributes(getContext(), i, attrs); + PAL = PAL.addAttribute(getContext(), i, Attr); setAttributes(PAL); } -void Function::removeAttributes(unsigned i, AttributeSet attrs) { +void Function::addAttributes(unsigned i, AttributeSet Attrs) { AttributeSet PAL = getAttributes(); - PAL = PAL.removeAttributes(getContext(), i, attrs); + PAL = PAL.addAttributes(getContext(), i, Attrs); + setAttributes(PAL); +} + +void Function::removeAttribute(unsigned i, Attribute::AttrKind Kind) { + AttributeSet PAL = getAttributes(); + PAL = PAL.removeAttribute(getContext(), i, Kind); + setAttributes(PAL); +} + +void Function::removeAttribute(unsigned i, StringRef Kind) { + AttributeSet PAL = getAttributes(); + PAL = PAL.removeAttribute(getContext(), i, Kind); + setAttributes(PAL); +} + +void Function::removeAttributes(unsigned i, AttributeSet Attrs) { + AttributeSet PAL = getAttributes(); + PAL = PAL.removeAttributes(getContext(), i, Attrs); setAttributes(PAL); } @@ -371,7 +419,7 @@ const std::string &Function::getGC() const { return getContext().getGC(*this); } -void Function::setGC(const std::string Str) { +void Function::setGC(std::string Str) { setValueSubclassDataBit(14, !Str.empty()); getContext().setGC(*this, std::move(Str)); } @@ -405,17 +453,58 @@ void Function::copyAttributesFrom(const GlobalValue *Src) { setPrologueData(SrcF->getPrologueData()); } +/// Table of string intrinsic names indexed by enum value. +static const char * const IntrinsicNameTable[] = { + "not_intrinsic", +#define GET_INTRINSIC_NAME_TABLE +#include "llvm/IR/Intrinsics.gen" +#undef GET_INTRINSIC_NAME_TABLE +}; + +/// Table of per-target intrinsic name tables. +#define GET_INTRINSIC_TARGET_DATA +#include "llvm/IR/Intrinsics.gen" +#undef GET_INTRINSIC_TARGET_DATA + +/// Find the segment of \c IntrinsicNameTable for intrinsics with the same +/// target as \c Name, or the generic table if \c Name is not target specific. +/// +/// Returns the relevant slice of \c IntrinsicNameTable +static ArrayRef<const char *> findTargetSubtable(StringRef Name) { + assert(Name.startswith("llvm.")); + + ArrayRef<IntrinsicTargetInfo> Targets(TargetInfos); + // Drop "llvm." and take the first dotted component. That will be the target + // if this is target specific. + StringRef Target = Name.drop_front(5).split('.').first; + auto It = std::lower_bound(Targets.begin(), Targets.end(), Target, + [](const IntrinsicTargetInfo &TI, + StringRef Target) { return TI.Name < Target; }); + // We've either found the target or just fall back to the generic set, which + // is always first. + const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0]; + return makeArrayRef(&IntrinsicNameTable[1] + TI.Offset, TI.Count); +} + /// \brief This does the actual lookup of an intrinsic ID which /// matches the given function name. static Intrinsic::ID lookupIntrinsicID(const ValueName *ValName) { - unsigned Len = ValName->getKeyLength(); - const char *Name = ValName->getKeyData(); + StringRef Name = ValName->getKey(); -#define GET_FUNCTION_RECOGNIZER -#include "llvm/IR/Intrinsics.gen" -#undef GET_FUNCTION_RECOGNIZER + ArrayRef<const char *> NameTable = findTargetSubtable(Name); + int Idx = Intrinsic::lookupLLVMIntrinsicByName(NameTable, Name); + if (Idx == -1) + return Intrinsic::not_intrinsic; + + // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have + // an index into a sub-table. + int Adjust = NameTable.data() - IntrinsicNameTable; + Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust); - return Intrinsic::not_intrinsic; + // If the intrinsic is not overloaded, require an exact match. If it is + // overloaded, require a prefix match. + bool IsPrefixMatch = Name.size() > strlen(NameTable[Idx]); + return IsPrefixMatch == isOverloaded(ID) ? ID : Intrinsic::not_intrinsic; } void Function::recalculateIntrinsicID() { @@ -470,17 +559,9 @@ static std::string getMangledTypeStr(Type* Ty) { std::string Intrinsic::getName(ID id, ArrayRef<Type*> Tys) { assert(id < num_intrinsics && "Invalid intrinsic ID!"); - static const char * const Table[] = { - "not_intrinsic", -#define GET_INTRINSIC_NAME_TABLE -#include "llvm/IR/Intrinsics.gen" -#undef GET_INTRINSIC_NAME_TABLE - }; - if (Tys.empty()) - return Table[id]; - std::string Result(Table[id]); - for (unsigned i = 0; i < Tys.size(); ++i) { - Result += "." + getMangledTypeStr(Tys[i]); + std::string Result(IntrinsicNameTable[id]); + for (Type *Ty : Tys) { + Result += "." + getMangledTypeStr(Ty); } return Result; } @@ -867,6 +948,198 @@ Function *Intrinsic::getDeclaration(Module *M, ID id, ArrayRef<Type*> Tys) { #include "llvm/IR/Intrinsics.gen" #undef GET_LLVM_INTRINSIC_FOR_MS_BUILTIN +bool Intrinsic::matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos, + SmallVectorImpl<Type*> &ArgTys) { + using namespace Intrinsic; + + // If we ran out of descriptors, there are too many arguments. + if (Infos.empty()) return true; + IITDescriptor D = Infos.front(); + Infos = Infos.slice(1); + + switch (D.Kind) { + case IITDescriptor::Void: return !Ty->isVoidTy(); + case IITDescriptor::VarArg: return true; + case IITDescriptor::MMX: return !Ty->isX86_MMXTy(); + case IITDescriptor::Token: return !Ty->isTokenTy(); + case IITDescriptor::Metadata: return !Ty->isMetadataTy(); + case IITDescriptor::Half: return !Ty->isHalfTy(); + case IITDescriptor::Float: return !Ty->isFloatTy(); + case IITDescriptor::Double: return !Ty->isDoubleTy(); + case IITDescriptor::Integer: return !Ty->isIntegerTy(D.Integer_Width); + case IITDescriptor::Vector: { + VectorType *VT = dyn_cast<VectorType>(Ty); + return !VT || VT->getNumElements() != D.Vector_Width || + matchIntrinsicType(VT->getElementType(), Infos, ArgTys); + } + case IITDescriptor::Pointer: { + PointerType *PT = dyn_cast<PointerType>(Ty); + return !PT || PT->getAddressSpace() != D.Pointer_AddressSpace || + matchIntrinsicType(PT->getElementType(), Infos, ArgTys); + } + + case IITDescriptor::Struct: { + StructType *ST = dyn_cast<StructType>(Ty); + if (!ST || ST->getNumElements() != D.Struct_NumElements) + return true; + + for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i) + if (matchIntrinsicType(ST->getElementType(i), Infos, ArgTys)) + return true; + return false; + } + + case IITDescriptor::Argument: + // Two cases here - If this is the second occurrence of an argument, verify + // that the later instance matches the previous instance. + if (D.getArgumentNumber() < ArgTys.size()) + return Ty != ArgTys[D.getArgumentNumber()]; + + // Otherwise, if this is the first instance of an argument, record it and + // verify the "Any" kind. + assert(D.getArgumentNumber() == ArgTys.size() && "Table consistency error"); + ArgTys.push_back(Ty); + + switch (D.getArgumentKind()) { + case IITDescriptor::AK_Any: return false; // Success + case IITDescriptor::AK_AnyInteger: return !Ty->isIntOrIntVectorTy(); + case IITDescriptor::AK_AnyFloat: return !Ty->isFPOrFPVectorTy(); + case IITDescriptor::AK_AnyVector: return !isa<VectorType>(Ty); + case IITDescriptor::AK_AnyPointer: return !isa<PointerType>(Ty); + } + llvm_unreachable("all argument kinds not covered"); + + case IITDescriptor::ExtendArgument: { + // This may only be used when referring to a previous vector argument. + if (D.getArgumentNumber() >= ArgTys.size()) + return true; + + Type *NewTy = ArgTys[D.getArgumentNumber()]; + if (VectorType *VTy = dyn_cast<VectorType>(NewTy)) + NewTy = VectorType::getExtendedElementVectorType(VTy); + else if (IntegerType *ITy = dyn_cast<IntegerType>(NewTy)) + NewTy = IntegerType::get(ITy->getContext(), 2 * ITy->getBitWidth()); + else + return true; + + return Ty != NewTy; + } + case IITDescriptor::TruncArgument: { + // This may only be used when referring to a previous vector argument. + if (D.getArgumentNumber() >= ArgTys.size()) + return true; + + Type *NewTy = ArgTys[D.getArgumentNumber()]; + if (VectorType *VTy = dyn_cast<VectorType>(NewTy)) + NewTy = VectorType::getTruncatedElementVectorType(VTy); + else if (IntegerType *ITy = dyn_cast<IntegerType>(NewTy)) + NewTy = IntegerType::get(ITy->getContext(), ITy->getBitWidth() / 2); + else + return true; + + return Ty != NewTy; + } + case IITDescriptor::HalfVecArgument: + // This may only be used when referring to a previous vector argument. + return D.getArgumentNumber() >= ArgTys.size() || + !isa<VectorType>(ArgTys[D.getArgumentNumber()]) || + VectorType::getHalfElementsVectorType( + cast<VectorType>(ArgTys[D.getArgumentNumber()])) != Ty; + case IITDescriptor::SameVecWidthArgument: { + if (D.getArgumentNumber() >= ArgTys.size()) + return true; + VectorType * ReferenceType = + dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]); + VectorType *ThisArgType = dyn_cast<VectorType>(Ty); + if (!ThisArgType || !ReferenceType || + (ReferenceType->getVectorNumElements() != + ThisArgType->getVectorNumElements())) + return true; + return matchIntrinsicType(ThisArgType->getVectorElementType(), + Infos, ArgTys); + } + case IITDescriptor::PtrToArgument: { + if (D.getArgumentNumber() >= ArgTys.size()) + return true; + Type * ReferenceType = ArgTys[D.getArgumentNumber()]; + PointerType *ThisArgType = dyn_cast<PointerType>(Ty); + return (!ThisArgType || ThisArgType->getElementType() != ReferenceType); + } + case IITDescriptor::VecOfPtrsToElt: { + if (D.getArgumentNumber() >= ArgTys.size()) + return true; + VectorType * ReferenceType = + dyn_cast<VectorType> (ArgTys[D.getArgumentNumber()]); + VectorType *ThisArgVecTy = dyn_cast<VectorType>(Ty); + if (!ThisArgVecTy || !ReferenceType || + (ReferenceType->getVectorNumElements() != + ThisArgVecTy->getVectorNumElements())) + return true; + PointerType *ThisArgEltTy = + dyn_cast<PointerType>(ThisArgVecTy->getVectorElementType()); + if (!ThisArgEltTy) + return true; + return ThisArgEltTy->getElementType() != + ReferenceType->getVectorElementType(); + } + } + llvm_unreachable("unhandled"); +} + +bool +Intrinsic::matchIntrinsicVarArg(bool isVarArg, + ArrayRef<Intrinsic::IITDescriptor> &Infos) { + // If there are no descriptors left, then it can't be a vararg. + if (Infos.empty()) + return isVarArg; + + // There should be only one descriptor remaining at this point. + if (Infos.size() != 1) + return true; + + // Check and verify the descriptor. + IITDescriptor D = Infos.front(); + Infos = Infos.slice(1); + if (D.Kind == IITDescriptor::VarArg) + return !isVarArg; + + return true; +} + +Optional<Function*> Intrinsic::remangleIntrinsicFunction(Function *F) { + Intrinsic::ID ID = F->getIntrinsicID(); + if (!ID) + return None; + + FunctionType *FTy = F->getFunctionType(); + // Accumulate an array of overloaded types for the given intrinsic + SmallVector<Type *, 4> ArgTys; + { + SmallVector<Intrinsic::IITDescriptor, 8> Table; + getIntrinsicInfoTableEntries(ID, Table); + ArrayRef<Intrinsic::IITDescriptor> TableRef = Table; + + // If we encounter any problems matching the signature with the descriptor + // just give up remangling. It's up to verifier to report the discrepancy. + if (Intrinsic::matchIntrinsicType(FTy->getReturnType(), TableRef, ArgTys)) + return None; + for (auto Ty : FTy->params()) + if (Intrinsic::matchIntrinsicType(Ty, TableRef, ArgTys)) + return None; + if (Intrinsic::matchIntrinsicVarArg(FTy->isVarArg(), TableRef)) + return None; + } + + StringRef Name = F->getName(); + if (Name == Intrinsic::getName(ID, ArgTys)) + return None; + + auto NewDecl = Intrinsic::getDeclaration(F->getParent(), ID, ArgTys); + NewDecl->setCallingConv(F->getCallingConv()); + assert(NewDecl->getFunctionType() == FTy && "Shouldn't change the signature"); + return NewDecl; +} + /// hasAddressTaken - returns true if there are any uses of this function /// other than direct calls or invokes to it. bool Function::hasAddressTaken(const User* *PutOffender) const { @@ -874,11 +1147,17 @@ bool Function::hasAddressTaken(const User* *PutOffender) const { const User *FU = U.getUser(); if (isa<BlockAddress>(FU)) continue; - if (!isa<CallInst>(FU) && !isa<InvokeInst>(FU)) - return PutOffender ? (*PutOffender = FU, true) : true; + if (!isa<CallInst>(FU) && !isa<InvokeInst>(FU)) { + if (PutOffender) + *PutOffender = FU; + return true; + } ImmutableCallSite CS(cast<Instruction>(FU)); - if (!CS.isCallee(&U)) - return PutOffender ? (*PutOffender = FU, true) : true; + if (!CS.isCallee(&U)) { + if (PutOffender) + *PutOffender = FU; + return true; + } } return false; } diff --git a/lib/IR/FunctionInfo.cpp b/lib/IR/FunctionInfo.cpp deleted file mode 100644 index 17a67bcf04721..0000000000000 --- a/lib/IR/FunctionInfo.cpp +++ /dev/null @@ -1,67 +0,0 @@ -//===-- FunctionInfo.cpp - Function Info Index ----------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file implements the function info index and summary classes for the -// IR library. -// -//===----------------------------------------------------------------------===// - -#include "llvm/IR/FunctionInfo.h" -#include "llvm/ADT/StringMap.h" -using namespace llvm; - -// Create the combined function index/summary from multiple -// per-module instances. -void FunctionInfoIndex::mergeFrom(std::unique_ptr<FunctionInfoIndex> Other, - uint64_t NextModuleId) { - - StringRef ModPath; - for (auto &OtherFuncInfoLists : *Other) { - std::string FuncName = OtherFuncInfoLists.getKey(); - FunctionInfoList &List = OtherFuncInfoLists.second; - - // Assert that the func info list only has one entry, since we shouldn't - // have duplicate names within a single per-module index. - assert(List.size() == 1); - std::unique_ptr<FunctionInfo> Info = std::move(List.front()); - - // Skip if there was no function summary section. - if (!Info->functionSummary()) - continue; - - // Add the module path string ref for this module if we haven't already - // saved a reference to it. - if (ModPath.empty()) - ModPath = - addModulePath(Info->functionSummary()->modulePath(), NextModuleId); - else - assert(ModPath == Info->functionSummary()->modulePath() && - "Each module in the combined map should have a unique ID"); - - // Note the module path string ref was copied above and is still owned by - // the original per-module index. Reset it to the new module path - // string reference owned by the combined index. - Info->functionSummary()->setModulePath(ModPath); - - // If it is a local function, rename it. - if (Info->functionSummary()->isLocalFunction()) { - // Any local functions are virtually renamed when being added to the - // combined index map, to disambiguate from other functions with - // the same name. The symbol table created for the combined index - // file should contain the renamed symbols. - FuncName = - FunctionInfoIndex::getGlobalNameForLocal(FuncName, NextModuleId); - } - - // Add new function info to existing list. There may be duplicates when - // combining FunctionMap entries, due to COMDAT functions. Any local - // functions were virtually renamed above. - addFunctionInfo(FuncName, std::move(Info)); - } -} diff --git a/lib/IR/GCOV.cpp b/lib/IR/GCOV.cpp index 35b8157751b6c..a9f7f45ee3059 100644 --- a/lib/IR/GCOV.cpp +++ b/lib/IR/GCOV.cpp @@ -22,6 +22,7 @@ #include "llvm/Support/raw_ostream.h" #include <algorithm> #include <system_error> + using namespace llvm; //===----------------------------------------------------------------------===// @@ -104,7 +105,7 @@ bool GCOVFile::readGCDA(GCOVBuffer &Buffer) { } /// dump - Dump GCOVFile content to dbgs() for debugging purposes. -void GCOVFile::dump() const { +LLVM_DUMP_METHOD void GCOVFile::dump() const { for (const auto &FPtr : Functions) FPtr->dump(); } @@ -247,10 +248,12 @@ bool GCOVFunction::readGCNO(GCOVBuffer &Buff, GCOV::GCOVVersion Version) { /// readGCDA - Read a function from the GCDA buffer. Return false if an error /// occurs. bool GCOVFunction::readGCDA(GCOVBuffer &Buff, GCOV::GCOVVersion Version) { - uint32_t Dummy; - if (!Buff.readInt(Dummy)) + uint32_t HeaderLength; + if (!Buff.readInt(HeaderLength)) return false; // Function header length + uint64_t EndPos = Buff.getCursor() + HeaderLength * sizeof(uint32_t); + uint32_t GCDAIdent; if (!Buff.readInt(GCDAIdent)) return false; @@ -280,13 +283,15 @@ bool GCOVFunction::readGCDA(GCOVBuffer &Buff, GCOV::GCOVVersion Version) { } } - StringRef GCDAName; - if (!Buff.readString(GCDAName)) - return false; - if (Name != GCDAName) { - errs() << "Function names do not match: " << Name << " != " << GCDAName - << ".\n"; - return false; + if (Buff.getCursor() < EndPos) { + StringRef GCDAName; + if (!Buff.readString(GCDAName)) + return false; + if (Name != GCDAName) { + errs() << "Function names do not match: " << Name << " != " << GCDAName + << ".\n"; + return false; + } } if (!Buff.readArcTag()) { @@ -340,7 +345,7 @@ uint64_t GCOVFunction::getExitCount() const { } /// dump - Dump GCOVFunction content to dbgs() for debugging purposes. -void GCOVFunction::dump() const { +LLVM_DUMP_METHOD void GCOVFunction::dump() const { dbgs() << "===== " << Name << " (" << Ident << ") @ " << Filename << ":" << LineNumber << "\n"; for (const auto &Block : Blocks) @@ -397,7 +402,7 @@ void GCOVBlock::collectLineCounts(FileInfo &FI) { } /// dump - Dump GCOVBlock content to dbgs() for debugging purposes. -void GCOVBlock::dump() const { +LLVM_DUMP_METHOD void GCOVBlock::dump() const { dbgs() << "Block : " << Number << " Counter : " << Counter << "\n"; if (!SrcEdges.empty()) { dbgs() << "\tSource Edges : "; @@ -496,7 +501,7 @@ public: OS << format("%5u:", LineNum) << Line << "\n"; } }; -} +} // end anonymous namespace /// Convert a path to a gcov filename. If PreservePaths is true, this /// translates "/" to "#", ".." to "^", and drops ".", to match gcov. @@ -683,7 +688,6 @@ void FileInfo::print(raw_ostream &InfoOS, StringRef MainFilename, if (Options.FuncCoverage) printFuncCoverage(InfoOS); printFileCoverage(InfoOS); - return; } /// printFunctionSummary - Print function and block summary. diff --git a/lib/IR/Globals.cpp b/lib/IR/Globals.cpp index a61b62bd9687e..6715484518a9d 100644 --- a/lib/IR/Globals.cpp +++ b/lib/IR/Globals.cpp @@ -43,7 +43,7 @@ void GlobalValue::destroyConstantImpl() { llvm_unreachable("You can't GV->destroyConstantImpl()!"); } -Value *GlobalValue::handleOperandChangeImpl(Value *From, Value *To, Use *U) { +Value *GlobalValue::handleOperandChangeImpl(Value *From, Value *To) { llvm_unreachable("Unsupported class for handleOperandChange()!"); } @@ -51,7 +51,7 @@ Value *GlobalValue::handleOperandChangeImpl(Value *From, Value *To, Use *U) { /// create a GlobalValue) from the GlobalValue Src to this one. void GlobalValue::copyAttributesFrom(const GlobalValue *Src) { setVisibility(Src->getVisibility()); - setUnnamedAddr(Src->hasUnnamedAddr()); + setUnnamedAddr(Src->getUnnamedAddr()); setDLLStorageClass(Src->getDLLStorageClass()); } @@ -81,13 +81,13 @@ void GlobalObject::setAlignment(unsigned Align) { unsigned GlobalObject::getGlobalObjectSubClassData() const { unsigned ValueData = getGlobalValueSubClassData(); - return ValueData >> AlignmentBits; + return ValueData >> GlobalObjectBits; } void GlobalObject::setGlobalObjectSubClassData(unsigned Val) { unsigned OldData = getGlobalValueSubClassData(); - setGlobalValueSubClassData((OldData & AlignmentMask) | - (Val << AlignmentBits)); + setGlobalValueSubClassData((OldData & GlobalObjectMask) | + (Val << GlobalObjectBits)); assert(getGlobalObjectSubClassData() == Val && "representation error"); } @@ -99,7 +99,36 @@ void GlobalObject::copyAttributesFrom(const GlobalValue *Src) { } } -const char *GlobalValue::getSection() const { +std::string GlobalValue::getGlobalIdentifier(StringRef Name, + GlobalValue::LinkageTypes Linkage, + StringRef FileName) { + + // Value names may be prefixed with a binary '1' to indicate + // that the backend should not modify the symbols due to any platform + // naming convention. Do not include that '1' in the PGO profile name. + if (Name[0] == '\1') + Name = Name.substr(1); + + std::string NewName = Name; + if (llvm::GlobalValue::isLocalLinkage(Linkage)) { + // For local symbols, prepend the main file name to distinguish them. + // Do not include the full path in the file name since there's no guarantee + // that it will stay the same, e.g., if the files are checked out from + // version control in different locations. + if (FileName.empty()) + NewName = NewName.insert(0, "<unknown>:"); + else + NewName = NewName.insert(0, FileName.str() + ":"); + } + return NewName; +} + +std::string GlobalValue::getGlobalIdentifier() const { + return getGlobalIdentifier(getName(), getLinkage(), + getParent()->getSourceFileName()); +} + +StringRef GlobalValue::getSection() const { if (auto *GA = dyn_cast<GlobalAlias>(this)) { // In general we cannot compute this at the IR level, but we try. if (const GlobalObject *GO = GA->getBaseObject()) @@ -116,10 +145,18 @@ Comdat *GlobalValue::getComdat() { return const_cast<GlobalObject *>(GO)->getComdat(); return nullptr; } + // ifunc and its resolver are separate things so don't use resolver comdat. + if (isa<GlobalIFunc>(this)) + return nullptr; return cast<GlobalObject>(this)->getComdat(); } -void GlobalObject::setSection(StringRef S) { Section = S; } +void GlobalObject::setSection(StringRef S) { + Section = S; + + // The C api requires this to be null terminated. + Section.c_str(); +} bool GlobalValue::isDeclaration() const { // Globals are definitions if they have an initializer. @@ -130,8 +167,8 @@ bool GlobalValue::isDeclaration() const { if (const Function *F = dyn_cast<Function>(this)) return F->empty() && !F->isMaterializable(); - // Aliases are always definitions. - assert(isa<GlobalAlias>(this)); + // Aliases and ifuncs are always definitions. + assert(isa<GlobalIndirectSymbol>(this)); return false; } @@ -242,7 +279,7 @@ void GlobalVariable::setInitializer(Constant *InitVal) { setGlobalVariableNumOperands(0); } } else { - assert(InitVal->getType() == getType()->getElementType() && + assert(InitVal->getType() == getValueType() && "Initializer type must match GlobalVariable type"); // Note, the num operands is used to compute the offset of the operand, so // the order here matters. We need to set num operands to 1 first so that @@ -263,6 +300,22 @@ void GlobalVariable::copyAttributesFrom(const GlobalValue *Src) { } } +void GlobalVariable::dropAllReferences() { + User::dropAllReferences(); + clearMetadata(); +} + +//===----------------------------------------------------------------------===// +// GlobalIndirectSymbol Implementation +//===----------------------------------------------------------------------===// + +GlobalIndirectSymbol::GlobalIndirectSymbol(Type *Ty, ValueTy VTy, + unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, + Constant *Symbol) + : GlobalValue(Ty, VTy, &Op<0>(), 1, Linkage, Name, AddressSpace) { + Op<0>() = Symbol; +} + //===----------------------------------------------------------------------===// // GlobalAlias Implementation @@ -271,10 +324,8 @@ void GlobalVariable::copyAttributesFrom(const GlobalValue *Src) { GlobalAlias::GlobalAlias(Type *Ty, unsigned AddressSpace, LinkageTypes Link, const Twine &Name, Constant *Aliasee, Module *ParentModule) - : GlobalValue(Ty, Value::GlobalAliasVal, &Op<0>(), 1, Link, Name, - AddressSpace) { - Op<0>() = Aliasee; - + : GlobalIndirectSymbol(Ty, Value::GlobalAliasVal, AddressSpace, Link, Name, + Aliasee) { if (ParentModule) ParentModule->getAliasList().push_back(this); } @@ -323,5 +374,36 @@ void GlobalAlias::eraseFromParent() { void GlobalAlias::setAliasee(Constant *Aliasee) { assert((!Aliasee || Aliasee->getType() == getType()) && "Alias and aliasee types should match!"); - setOperand(0, Aliasee); + setIndirectSymbol(Aliasee); +} + +//===----------------------------------------------------------------------===// +// GlobalIFunc Implementation +//===----------------------------------------------------------------------===// + +GlobalIFunc::GlobalIFunc(Type *Ty, unsigned AddressSpace, LinkageTypes Link, + const Twine &Name, Constant *Resolver, + Module *ParentModule) + : GlobalIndirectSymbol(Ty, Value::GlobalIFuncVal, AddressSpace, Link, Name, + Resolver) { + if (ParentModule) + ParentModule->getIFuncList().push_back(this); +} + +GlobalIFunc *GlobalIFunc::create(Type *Ty, unsigned AddressSpace, + LinkageTypes Link, const Twine &Name, + Constant *Resolver, Module *ParentModule) { + return new GlobalIFunc(Ty, AddressSpace, Link, Name, Resolver, ParentModule); +} + +void GlobalIFunc::setParent(Module *parent) { + Parent = parent; +} + +void GlobalIFunc::removeFromParent() { + getParent()->getIFuncList().remove(getIterator()); +} + +void GlobalIFunc::eraseFromParent() { + getParent()->getIFuncList().erase(getIterator()); } diff --git a/lib/IR/IRBuilder.cpp b/lib/IR/IRBuilder.cpp index 447412936335c..298331d51c884 100644 --- a/lib/IR/IRBuilder.cpp +++ b/lib/IR/IRBuilder.cpp @@ -34,7 +34,7 @@ GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str, StrConstant, Name, nullptr, GlobalVariable::NotThreadLocal, AddressSpace); - GV->setUnnamedAddr(true); + GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); return GV; } @@ -201,52 +201,113 @@ CallInst *IRBuilderBase::CreateAssumption(Value *Cond) { return createCallHelper(FnAssume, Ops, this); } -/// Create a call to a Masked Load intrinsic. -/// Ptr - the base pointer for the load -/// Align - alignment of the source location -/// Mask - an vector of booleans which indicates what vector lanes should -/// be accessed in memory -/// PassThru - a pass-through value that is used to fill the masked-off lanes -/// of the result -/// Name - name of the result variable +/// \brief Create a call to a Masked Load intrinsic. +/// \p Ptr - base pointer for the load +/// \p Align - alignment of the source location +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory +/// \p PassThru - pass-through value that is used to fill the masked-off lanes +/// of the result +/// \p Name - name of the result variable CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align, Value *Mask, Value *PassThru, const Twine &Name) { - assert(Ptr->getType()->isPointerTy() && "Ptr must be of pointer type"); - // DataTy is the overloaded type - Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType(); + PointerType *PtrTy = cast<PointerType>(Ptr->getType()); + Type *DataTy = PtrTy->getElementType(); assert(DataTy->isVectorTy() && "Ptr should point to a vector"); if (!PassThru) PassThru = UndefValue::get(DataTy); + Type *OverloadedTypes[] = { DataTy, PtrTy }; Value *Ops[] = { Ptr, getInt32(Align), Mask, PassThru}; - return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, DataTy, Name); + return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, + OverloadedTypes, Name); } -/// Create a call to a Masked Store intrinsic. -/// Val - the data to be stored, -/// Ptr - the base pointer for the store -/// Align - alignment of the destination location -/// Mask - an vector of booleans which indicates what vector lanes should -/// be accessed in memory +/// \brief Create a call to a Masked Store intrinsic. +/// \p Val - data to be stored, +/// \p Ptr - base pointer for the store +/// \p Align - alignment of the destination location +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align, Value *Mask) { + PointerType *PtrTy = cast<PointerType>(Ptr->getType()); + Type *DataTy = PtrTy->getElementType(); + assert(DataTy->isVectorTy() && "Ptr should point to a vector"); + Type *OverloadedTypes[] = { DataTy, PtrTy }; Value *Ops[] = { Val, Ptr, getInt32(Align), Mask }; - // Type of the data to be stored - the only one overloaded type - return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, Val->getType()); + return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes); } /// Create a call to a Masked intrinsic, with given intrinsic Id, -/// an array of operands - Ops, and one overloaded type - DataTy +/// an array of operands - Ops, and an array of overloaded types - +/// OverloadedTypes. CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops, - Type *DataTy, + ArrayRef<Type *> OverloadedTypes, const Twine &Name) { Module *M = BB->getParent()->getParent(); - Type *OverloadedTypes[] = { DataTy }; Value *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes); return createCallHelper(TheFn, Ops, this, Name); } +/// \brief Create a call to a Masked Gather intrinsic. +/// \p Ptrs - vector of pointers for loading +/// \p Align - alignment for one element +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory +/// \p PassThru - pass-through value that is used to fill the masked-off lanes +/// of the result +/// \p Name - name of the result variable +CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align, + Value *Mask, Value *PassThru, + const Twine& Name) { + auto PtrsTy = cast<VectorType>(Ptrs->getType()); + auto PtrTy = cast<PointerType>(PtrsTy->getElementType()); + unsigned NumElts = PtrsTy->getVectorNumElements(); + Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts); + + if (!Mask) + Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context), + NumElts)); + + Value * Ops[] = {Ptrs, getInt32(Align), Mask, UndefValue::get(DataTy)}; + + // We specify only one type when we create this intrinsic. Types of other + // arguments are derived from this type. + return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, { DataTy }, Name); +} + +/// \brief Create a call to a Masked Scatter intrinsic. +/// \p Data - data to be stored, +/// \p Ptrs - the vector of pointers, where the \p Data elements should be +/// stored +/// \p Align - alignment for one element +/// \p Mask - vector of booleans which indicates what vector lanes should +/// be accessed in memory +CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs, + unsigned Align, Value *Mask) { + auto PtrsTy = cast<VectorType>(Ptrs->getType()); + auto DataTy = cast<VectorType>(Data->getType()); + unsigned NumElts = PtrsTy->getVectorNumElements(); + +#ifndef NDEBUG + auto PtrTy = cast<PointerType>(PtrsTy->getElementType()); + assert(NumElts == DataTy->getVectorNumElements() && + PtrTy->getElementType() == DataTy->getElementType() && + "Incompatible pointer and data types"); +#endif + + if (!Mask) + Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context), + NumElts)); + Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask}; + + // We specify only one type when we create this intrinsic. Types of other + // arguments are derived from this type. + return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, { DataTy }); +} + template <typename T0, typename T1, typename T2, typename T3> static std::vector<Value *> getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes, diff --git a/lib/IR/IRPrintingPasses.cpp b/lib/IR/IRPrintingPasses.cpp index 822dbeb08b330..4d2f9b98911b1 100644 --- a/lib/IR/IRPrintingPasses.cpp +++ b/lib/IR/IRPrintingPasses.cpp @@ -26,7 +26,7 @@ PrintModulePass::PrintModulePass(raw_ostream &OS, const std::string &Banner, : OS(OS), Banner(Banner), ShouldPreserveUseListOrder(ShouldPreserveUseListOrder) {} -PreservedAnalyses PrintModulePass::run(Module &M) { +PreservedAnalyses PrintModulePass::run(Module &M, AnalysisManager<Module> &) { OS << Banner; if (llvm::isFunctionInPrintList("*")) M.print(OS, nullptr, ShouldPreserveUseListOrder); @@ -42,7 +42,8 @@ PrintFunctionPass::PrintFunctionPass() : OS(dbgs()) {} PrintFunctionPass::PrintFunctionPass(raw_ostream &OS, const std::string &Banner) : OS(OS), Banner(Banner) {} -PreservedAnalyses PrintFunctionPass::run(Function &F) { +PreservedAnalyses PrintFunctionPass::run(Function &F, + AnalysisManager<Function> &) { if (isFunctionInPrintList(F.getName())) OS << Banner << static_cast<Value &>(F); return PreservedAnalyses::all(); @@ -61,7 +62,8 @@ public: : ModulePass(ID), P(OS, Banner, ShouldPreserveUseListOrder) {} bool runOnModule(Module &M) override { - P.run(M); + ModuleAnalysisManager DummyMAM; + P.run(M, DummyMAM); return false; } @@ -81,7 +83,8 @@ public: // This pass just prints a banner followed by the function as it's processed. bool runOnFunction(Function &F) override { - P.run(F); + FunctionAnalysisManager DummyFAM; + P.run(F, DummyFAM); return false; } diff --git a/lib/IR/InlineAsm.cpp b/lib/IR/InlineAsm.cpp index 15d3b830b8fc7..d6cf8c543dbdb 100644 --- a/lib/IR/InlineAsm.cpp +++ b/lib/IR/InlineAsm.cpp @@ -142,14 +142,14 @@ bool InlineAsm::ConstraintInfo::Parse(StringRef Str, // Find the end of the register name. StringRef::iterator ConstraintEnd = std::find(I+1, E, '}'); if (ConstraintEnd == E) return true; // "{foo" - pCodes->push_back(std::string(I, ConstraintEnd+1)); + pCodes->push_back(StringRef(I, ConstraintEnd+1 - I)); I = ConstraintEnd+1; } else if (isdigit(static_cast<unsigned char>(*I))) { // Matching Constraint // Maximal munch numbers. StringRef::iterator NumStart = I; while (I != E && isdigit(static_cast<unsigned char>(*I))) ++I; - pCodes->push_back(std::string(NumStart, I)); + pCodes->push_back(StringRef(NumStart, I - NumStart)); unsigned N = atoi(pCodes->back().c_str()); // Check that this is a valid matching constraint! if (N >= ConstraintsSoFar.size() || ConstraintsSoFar[N].Type != isOutput|| @@ -183,11 +183,11 @@ bool InlineAsm::ConstraintInfo::Parse(StringRef Str, } else if (*I == '^') { // Multi-letter constraint // FIXME: For now assuming these are 2-character constraints. - pCodes->push_back(std::string(I+1, I+3)); + pCodes->push_back(StringRef(I+1, 2)); I += 3; } else { // Single letter constraint. - pCodes->push_back(std::string(I, I+1)); + pCodes->push_back(StringRef(I, 1)); ++I; } } diff --git a/lib/IR/Instruction.cpp b/lib/IR/Instruction.cpp index 4b33d2e66ea19..ed08f85c60b6d 100644 --- a/lib/IR/Instruction.cpp +++ b/lib/IR/Instruction.cpp @@ -96,6 +96,30 @@ void Instruction::moveBefore(Instruction *MovePos) { MovePos->getIterator(), getParent()->getInstList(), getIterator()); } +void Instruction::setHasNoUnsignedWrap(bool b) { + cast<OverflowingBinaryOperator>(this)->setHasNoUnsignedWrap(b); +} + +void Instruction::setHasNoSignedWrap(bool b) { + cast<OverflowingBinaryOperator>(this)->setHasNoSignedWrap(b); +} + +void Instruction::setIsExact(bool b) { + cast<PossiblyExactOperator>(this)->setIsExact(b); +} + +bool Instruction::hasNoUnsignedWrap() const { + return cast<OverflowingBinaryOperator>(this)->hasNoUnsignedWrap(); +} + +bool Instruction::hasNoSignedWrap() const { + return cast<OverflowingBinaryOperator>(this)->hasNoSignedWrap(); +} + +bool Instruction::isExact() const { + return cast<PossiblyExactOperator>(this)->isExact(); +} + /// Set or clear the unsafe-algebra flag on this instruction, which must be an /// operator which supports this flag. See LangRef.html for the meaning of this /// flag. @@ -190,6 +214,54 @@ void Instruction::copyFastMathFlags(const Instruction *I) { copyFastMathFlags(I->getFastMathFlags()); } +void Instruction::copyIRFlags(const Value *V) { + // Copy the wrapping flags. + if (auto *OB = dyn_cast<OverflowingBinaryOperator>(V)) { + if (isa<OverflowingBinaryOperator>(this)) { + setHasNoSignedWrap(OB->hasNoSignedWrap()); + setHasNoUnsignedWrap(OB->hasNoUnsignedWrap()); + } + } + + // Copy the exact flag. + if (auto *PE = dyn_cast<PossiblyExactOperator>(V)) + if (isa<PossiblyExactOperator>(this)) + setIsExact(PE->isExact()); + + // Copy the fast-math flags. + if (auto *FP = dyn_cast<FPMathOperator>(V)) + if (isa<FPMathOperator>(this)) + copyFastMathFlags(FP->getFastMathFlags()); + + if (auto *SrcGEP = dyn_cast<GetElementPtrInst>(V)) + if (auto *DestGEP = dyn_cast<GetElementPtrInst>(this)) + DestGEP->setIsInBounds(SrcGEP->isInBounds() | DestGEP->isInBounds()); +} + +void Instruction::andIRFlags(const Value *V) { + if (auto *OB = dyn_cast<OverflowingBinaryOperator>(V)) { + if (isa<OverflowingBinaryOperator>(this)) { + setHasNoSignedWrap(hasNoSignedWrap() & OB->hasNoSignedWrap()); + setHasNoUnsignedWrap(hasNoUnsignedWrap() & OB->hasNoUnsignedWrap()); + } + } + + if (auto *PE = dyn_cast<PossiblyExactOperator>(V)) + if (isa<PossiblyExactOperator>(this)) + setIsExact(isExact() & PE->isExact()); + + if (auto *FP = dyn_cast<FPMathOperator>(V)) { + if (isa<FPMathOperator>(this)) { + FastMathFlags FM = getFastMathFlags(); + FM &= FP->getFastMathFlags(); + copyFastMathFlags(FM); + } + } + + if (auto *SrcGEP = dyn_cast<GetElementPtrInst>(V)) + if (auto *DestGEP = dyn_cast<GetElementPtrInst>(this)) + DestGEP->setIsInBounds(SrcGEP->isInBounds() & DestGEP->isInBounds()); +} const char *Instruction::getOpcodeName(unsigned OpCode) { switch (OpCode) { @@ -271,13 +343,18 @@ const char *Instruction::getOpcodeName(unsigned OpCode) { } } -/// Return true if both instructions have the same special state -/// This must be kept in sync with lib/Transforms/IPO/MergeFunctions.cpp. +/// Return true if both instructions have the same special state This must be +/// kept in sync with FunctionComparator::cmpOperations in +/// lib/Transforms/IPO/MergeFunctions.cpp. static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2, bool IgnoreAlignment = false) { assert(I1->getOpcode() == I2->getOpcode() && "Can not compare special state of different instructions"); + if (const AllocaInst *AI = dyn_cast<AllocaInst>(I1)) + return AI->getAllocatedType() == cast<AllocaInst>(I2)->getAllocatedType() && + (AI->getAlignment() == cast<AllocaInst>(I2)->getAlignment() || + IgnoreAlignment); if (const LoadInst *LI = dyn_cast<LoadInst>(I1)) return LI->isVolatile() == cast<LoadInst>(I2)->isVolatile() && (LI->getAlignment() == cast<LoadInst>(I2)->getAlignment() || @@ -360,8 +437,7 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const { return haveSameSpecialState(this, I); } -// isSameOperationAs -// This should be kept in sync with isEquivalentOperation in +// Keep this in sync with FunctionComparator::cmpOperations in // lib/Transforms/IPO/MergeFunctions.cpp. bool Instruction::isSameOperationAs(const Instruction *I, unsigned flags) const { @@ -461,9 +537,9 @@ bool Instruction::isAtomic() const { case Instruction::Fence: return true; case Instruction::Load: - return cast<LoadInst>(this)->getOrdering() != NotAtomic; + return cast<LoadInst>(this)->getOrdering() != AtomicOrdering::NotAtomic; case Instruction::Store: - return cast<StoreInst>(this)->getOrdering() != NotAtomic; + return cast<StoreInst>(this)->getOrdering() != AtomicOrdering::NotAtomic; } } @@ -477,12 +553,6 @@ bool Instruction::mayThrow() const { return isa<ResumeInst>(this); } -bool Instruction::mayReturn() const { - if (const CallInst *CI = dyn_cast<CallInst>(this)) - return !CI->doesNotReturn(); - return true; -} - /// isAssociative - Return true if the instruction is associative: /// /// Associative operators satisfy: x op (y op z) === (x op y) op z diff --git a/lib/IR/Instructions.cpp b/lib/IR/Instructions.cpp index 7c64ca7b72753..b9c693ff19ad3 100644 --- a/lib/IR/Instructions.cpp +++ b/lib/IR/Instructions.cpp @@ -154,6 +154,24 @@ Value *PHINode::hasConstantValue() const { return ConstantValue; } +/// hasConstantOrUndefValue - Whether the specified PHI node always merges +/// together the same value, assuming that undefs result in the same value as +/// non-undefs. +/// Unlike \ref hasConstantValue, this does not return a value because the +/// unique non-undef incoming value need not dominate the PHI node. +bool PHINode::hasConstantOrUndefValue() const { + Value *ConstantValue = nullptr; + for (unsigned i = 0, e = getNumIncomingValues(); i != e; ++i) { + Value *Incoming = getIncomingValue(i); + if (Incoming != this && !isa<UndefValue>(Incoming)) { + if (ConstantValue && ConstantValue != Incoming) + return false; + ConstantValue = Incoming; + } + } + return true; +} + //===----------------------------------------------------------------------===// // LandingPadInst Implementation //===----------------------------------------------------------------------===// @@ -309,12 +327,26 @@ CallInst *CallInst::Create(CallInst *CI, ArrayRef<OperandBundleDef> OpB, NewCI->setCallingConv(CI->getCallingConv()); NewCI->SubclassOptionalData = CI->SubclassOptionalData; NewCI->setAttributes(CI->getAttributes()); + NewCI->setDebugLoc(CI->getDebugLoc()); return NewCI; } -void CallInst::addAttribute(unsigned i, Attribute::AttrKind attr) { +Value *CallInst::getReturnedArgOperand() const { + unsigned Index; + + if (AttributeList.hasAttrSomewhere(Attribute::Returned, &Index) && Index) + return getArgOperand(Index-1); + if (const Function *F = getCalledFunction()) + if (F->getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) && + Index) + return getArgOperand(Index-1); + + return nullptr; +} + +void CallInst::addAttribute(unsigned i, Attribute::AttrKind Kind) { AttributeSet PAL = getAttributes(); - PAL = PAL.addAttribute(getContext(), i, attr); + PAL = PAL.addAttribute(getContext(), i, Kind); setAttributes(PAL); } @@ -324,9 +356,27 @@ void CallInst::addAttribute(unsigned i, StringRef Kind, StringRef Value) { setAttributes(PAL); } -void CallInst::removeAttribute(unsigned i, Attribute attr) { +void CallInst::addAttribute(unsigned i, Attribute Attr) { + AttributeSet PAL = getAttributes(); + PAL = PAL.addAttribute(getContext(), i, Attr); + setAttributes(PAL); +} + +void CallInst::removeAttribute(unsigned i, Attribute::AttrKind Kind) { + AttributeSet PAL = getAttributes(); + PAL = PAL.removeAttribute(getContext(), i, Kind); + setAttributes(PAL); +} + +void CallInst::removeAttribute(unsigned i, StringRef Kind) { AttributeSet PAL = getAttributes(); - AttrBuilder B(attr); + PAL = PAL.removeAttribute(getContext(), i, Kind); + setAttributes(PAL); +} + +void CallInst::removeAttribute(unsigned i, Attribute Attr) { + AttributeSet PAL = getAttributes(); + AttrBuilder B(Attr); LLVMContext &Context = getContext(); PAL = PAL.removeAttributes(Context, i, AttributeSet::get(Context, i, B)); @@ -345,19 +395,26 @@ void CallInst::addDereferenceableOrNullAttr(unsigned i, uint64_t Bytes) { setAttributes(PAL); } -bool CallInst::paramHasAttr(unsigned i, Attribute::AttrKind A) const { +bool CallInst::paramHasAttr(unsigned i, Attribute::AttrKind Kind) const { assert(i < (getNumArgOperands() + 1) && "Param index out of bounds!"); - if (AttributeList.hasAttribute(i, A)) + if (AttributeList.hasAttribute(i, Kind)) return true; if (const Function *F = getCalledFunction()) - return F->getAttributes().hasAttribute(i, A); + return F->getAttributes().hasAttribute(i, Kind); return false; } -bool CallInst::dataOperandHasImpliedAttr(unsigned i, - Attribute::AttrKind A) const { +Attribute CallInst::getAttribute(unsigned i, Attribute::AttrKind Kind) const { + return getAttributes().getAttribute(i, Kind); +} +Attribute CallInst::getAttribute(unsigned i, StringRef Kind) const { + return getAttributes().getAttribute(i, Kind); +} + +bool CallInst::dataOperandHasImpliedAttr(unsigned i, + Attribute::AttrKind Kind) const { // There are getNumOperands() - 1 data operands. The last operand is the // callee. assert(i < getNumOperands() && "Data operand index out of bounds!"); @@ -367,11 +424,11 @@ bool CallInst::dataOperandHasImpliedAttr(unsigned i, // containing operand bundle, if the operand is a bundle operand. if (i < (getNumArgOperands() + 1)) - return paramHasAttr(i, A); + return paramHasAttr(i, Kind); assert(hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && "Must be either a call argument or an operand bundle!"); - return bundleOperandHasAttr(i - 1, A); + return bundleOperandHasAttr(i - 1, Kind); } /// IsConstantOne - Return true only if val is constant int 1 @@ -383,16 +440,17 @@ static bool IsConstantOne(Value *val) { static Instruction *createMalloc(Instruction *InsertBefore, BasicBlock *InsertAtEnd, Type *IntPtrTy, - Type *AllocTy, Value *AllocSize, - Value *ArraySize, Function *MallocF, - const Twine &Name) { + Type *AllocTy, Value *AllocSize, + Value *ArraySize, + ArrayRef<OperandBundleDef> OpB, + Function *MallocF, const Twine &Name) { assert(((!InsertBefore && InsertAtEnd) || (InsertBefore && !InsertAtEnd)) && "createMalloc needs either InsertBefore or InsertAtEnd"); // malloc(type) becomes: // bitcast (i8* malloc(typeSize)) to type* // malloc(type, arraySize) becomes: - // bitcast (i8 *malloc(typeSize*arraySize)) to type* + // bitcast (i8* malloc(typeSize*arraySize)) to type* if (!ArraySize) ArraySize = ConstantInt::get(IntPtrTy, 1); else if (ArraySize->getType() != IntPtrTy) { @@ -425,8 +483,8 @@ static Instruction *createMalloc(Instruction *InsertBefore, assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size"); // Create the call to Malloc. - BasicBlock* BB = InsertBefore ? InsertBefore->getParent() : InsertAtEnd; - Module* M = BB->getParent()->getParent(); + BasicBlock *BB = InsertBefore ? InsertBefore->getParent() : InsertAtEnd; + Module *M = BB->getParent()->getParent(); Type *BPTy = Type::getInt8PtrTy(BB->getContext()); Value *MallocFunc = MallocF; if (!MallocFunc) @@ -436,13 +494,14 @@ static Instruction *createMalloc(Instruction *InsertBefore, CallInst *MCall = nullptr; Instruction *Result = nullptr; if (InsertBefore) { - MCall = CallInst::Create(MallocFunc, AllocSize, "malloccall", InsertBefore); + MCall = CallInst::Create(MallocFunc, AllocSize, OpB, "malloccall", + InsertBefore); Result = MCall; if (Result->getType() != AllocPtrType) // Create a cast instruction to convert to the right type... Result = new BitCastInst(MCall, AllocPtrType, Name, InsertBefore); } else { - MCall = CallInst::Create(MallocFunc, AllocSize, "malloccall"); + MCall = CallInst::Create(MallocFunc, AllocSize, OpB, "malloccall"); Result = MCall; if (Result->getType() != AllocPtrType) { InsertAtEnd->getInstList().push_back(MCall); @@ -469,11 +528,21 @@ static Instruction *createMalloc(Instruction *InsertBefore, Instruction *CallInst::CreateMalloc(Instruction *InsertBefore, Type *IntPtrTy, Type *AllocTy, Value *AllocSize, Value *ArraySize, - Function * MallocF, + Function *MallocF, const Twine &Name) { return createMalloc(InsertBefore, nullptr, IntPtrTy, AllocTy, AllocSize, - ArraySize, MallocF, Name); + ArraySize, None, MallocF, Name); } +Instruction *CallInst::CreateMalloc(Instruction *InsertBefore, + Type *IntPtrTy, Type *AllocTy, + Value *AllocSize, Value *ArraySize, + ArrayRef<OperandBundleDef> OpB, + Function *MallocF, + const Twine &Name) { + return createMalloc(InsertBefore, nullptr, IntPtrTy, AllocTy, AllocSize, + ArraySize, OpB, MallocF, Name); +} + /// CreateMalloc - Generate the IR for a call to malloc: /// 1. Compute the malloc call's argument as the specified type's size, @@ -488,33 +557,43 @@ Instruction *CallInst::CreateMalloc(BasicBlock *InsertAtEnd, Value *AllocSize, Value *ArraySize, Function *MallocF, const Twine &Name) { return createMalloc(nullptr, InsertAtEnd, IntPtrTy, AllocTy, AllocSize, - ArraySize, MallocF, Name); + ArraySize, None, MallocF, Name); +} +Instruction *CallInst::CreateMalloc(BasicBlock *InsertAtEnd, + Type *IntPtrTy, Type *AllocTy, + Value *AllocSize, Value *ArraySize, + ArrayRef<OperandBundleDef> OpB, + Function *MallocF, const Twine &Name) { + return createMalloc(nullptr, InsertAtEnd, IntPtrTy, AllocTy, AllocSize, + ArraySize, OpB, MallocF, Name); } -static Instruction* createFree(Value* Source, Instruction *InsertBefore, +static Instruction *createFree(Value *Source, + ArrayRef<OperandBundleDef> Bundles, + Instruction *InsertBefore, BasicBlock *InsertAtEnd) { assert(((!InsertBefore && InsertAtEnd) || (InsertBefore && !InsertAtEnd)) && "createFree needs either InsertBefore or InsertAtEnd"); assert(Source->getType()->isPointerTy() && "Can not free something of nonpointer type!"); - BasicBlock* BB = InsertBefore ? InsertBefore->getParent() : InsertAtEnd; - Module* M = BB->getParent()->getParent(); + BasicBlock *BB = InsertBefore ? InsertBefore->getParent() : InsertAtEnd; + Module *M = BB->getParent()->getParent(); Type *VoidTy = Type::getVoidTy(M->getContext()); Type *IntPtrTy = Type::getInt8PtrTy(M->getContext()); // prototype free as "void free(void*)" Value *FreeFunc = M->getOrInsertFunction("free", VoidTy, IntPtrTy, nullptr); - CallInst* Result = nullptr; + CallInst *Result = nullptr; Value *PtrCast = Source; if (InsertBefore) { if (Source->getType() != IntPtrTy) PtrCast = new BitCastInst(Source, IntPtrTy, "", InsertBefore); - Result = CallInst::Create(FreeFunc, PtrCast, "", InsertBefore); + Result = CallInst::Create(FreeFunc, PtrCast, Bundles, "", InsertBefore); } else { if (Source->getType() != IntPtrTy) PtrCast = new BitCastInst(Source, IntPtrTy, "", InsertAtEnd); - Result = CallInst::Create(FreeFunc, PtrCast, ""); + Result = CallInst::Create(FreeFunc, PtrCast, Bundles, ""); } Result->setTailCall(); if (Function *F = dyn_cast<Function>(FreeFunc)) @@ -524,15 +603,27 @@ static Instruction* createFree(Value* Source, Instruction *InsertBefore, } /// CreateFree - Generate the IR for a call to the builtin free function. -Instruction * CallInst::CreateFree(Value* Source, Instruction *InsertBefore) { - return createFree(Source, InsertBefore, nullptr); +Instruction *CallInst::CreateFree(Value *Source, Instruction *InsertBefore) { + return createFree(Source, None, InsertBefore, nullptr); +} +Instruction *CallInst::CreateFree(Value *Source, + ArrayRef<OperandBundleDef> Bundles, + Instruction *InsertBefore) { + return createFree(Source, Bundles, InsertBefore, nullptr); } /// CreateFree - Generate the IR for a call to the builtin free function. /// Note: This function does not add the call to the basic block, that is the /// responsibility of the caller. -Instruction* CallInst::CreateFree(Value* Source, BasicBlock *InsertAtEnd) { - Instruction* FreeCall = createFree(Source, nullptr, InsertAtEnd); +Instruction *CallInst::CreateFree(Value *Source, BasicBlock *InsertAtEnd) { + Instruction *FreeCall = createFree(Source, None, nullptr, InsertAtEnd); + assert(FreeCall && "CreateFree did not create a CallInst"); + return FreeCall; +} +Instruction *CallInst::CreateFree(Value *Source, + ArrayRef<OperandBundleDef> Bundles, + BasicBlock *InsertAtEnd) { + Instruction *FreeCall = createFree(Source, Bundles, nullptr, InsertAtEnd); assert(FreeCall && "CreateFree did not create a CallInst"); return FreeCall; } @@ -596,6 +687,7 @@ InvokeInst *InvokeInst::Create(InvokeInst *II, ArrayRef<OperandBundleDef> OpB, NewII->setCallingConv(II->getCallingConv()); NewII->SubclassOptionalData = II->SubclassOptionalData; NewII->setAttributes(II->getAttributes()); + NewII->setDebugLoc(II->getDebugLoc()); return NewII; } @@ -609,18 +701,31 @@ void InvokeInst::setSuccessorV(unsigned idx, BasicBlock *B) { return setSuccessor(idx, B); } -bool InvokeInst::paramHasAttr(unsigned i, Attribute::AttrKind A) const { +Value *InvokeInst::getReturnedArgOperand() const { + unsigned Index; + + if (AttributeList.hasAttrSomewhere(Attribute::Returned, &Index) && Index) + return getArgOperand(Index-1); + if (const Function *F = getCalledFunction()) + if (F->getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) && + Index) + return getArgOperand(Index-1); + + return nullptr; +} + +bool InvokeInst::paramHasAttr(unsigned i, Attribute::AttrKind Kind) const { assert(i < (getNumArgOperands() + 1) && "Param index out of bounds!"); - if (AttributeList.hasAttribute(i, A)) + if (AttributeList.hasAttribute(i, Kind)) return true; if (const Function *F = getCalledFunction()) - return F->getAttributes().hasAttribute(i, A); + return F->getAttributes().hasAttribute(i, Kind); return false; } bool InvokeInst::dataOperandHasImpliedAttr(unsigned i, - Attribute::AttrKind A) const { + Attribute::AttrKind Kind) const { // There are getNumOperands() - 3 data operands. The last three operands are // the callee and the two successor basic blocks. assert(i < (getNumOperands() - 2) && "Data operand index out of bounds!"); @@ -630,27 +735,54 @@ bool InvokeInst::dataOperandHasImpliedAttr(unsigned i, // containing operand bundle, if the operand is a bundle operand. if (i < (getNumArgOperands() + 1)) - return paramHasAttr(i, A); + return paramHasAttr(i, Kind); assert(hasOperandBundles() && i >= (getBundleOperandsStartIndex() + 1) && "Must be either an invoke argument or an operand bundle!"); - return bundleOperandHasAttr(i - 1, A); + return bundleOperandHasAttr(i - 1, Kind); +} + +void InvokeInst::addAttribute(unsigned i, Attribute::AttrKind Kind) { + AttributeSet PAL = getAttributes(); + PAL = PAL.addAttribute(getContext(), i, Kind); + setAttributes(PAL); +} + +void InvokeInst::addAttribute(unsigned i, Attribute Attr) { + AttributeSet PAL = getAttributes(); + PAL = PAL.addAttribute(getContext(), i, Attr); + setAttributes(PAL); +} + +void InvokeInst::removeAttribute(unsigned i, Attribute::AttrKind Kind) { + AttributeSet PAL = getAttributes(); + PAL = PAL.removeAttribute(getContext(), i, Kind); + setAttributes(PAL); } -void InvokeInst::addAttribute(unsigned i, Attribute::AttrKind attr) { +void InvokeInst::removeAttribute(unsigned i, StringRef Kind) { AttributeSet PAL = getAttributes(); - PAL = PAL.addAttribute(getContext(), i, attr); + PAL = PAL.removeAttribute(getContext(), i, Kind); setAttributes(PAL); } -void InvokeInst::removeAttribute(unsigned i, Attribute attr) { +void InvokeInst::removeAttribute(unsigned i, Attribute Attr) { AttributeSet PAL = getAttributes(); - AttrBuilder B(attr); + AttrBuilder B(Attr); PAL = PAL.removeAttributes(getContext(), i, AttributeSet::get(getContext(), i, B)); setAttributes(PAL); } +Attribute InvokeInst::getAttribute(unsigned i, + Attribute::AttrKind Kind) const { + return getAttributes().getAttribute(i, Kind); +} + +Attribute InvokeInst::getAttribute(unsigned i, StringRef Kind) const { + return getAttributes().getAttribute(i, Kind); +} + void InvokeInst::addDereferenceableAttr(unsigned i, uint64_t Bytes) { AttributeSet PAL = getAttributes(); PAL = PAL.addDereferenceableAttr(getContext(), i, Bytes); @@ -1207,13 +1339,13 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, Instruction *InsertBef) - : LoadInst(Ty, Ptr, Name, isVolatile, Align, NotAtomic, CrossThread, - InsertBef) {} + : LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic, + CrossThread, InsertBef) {} LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, BasicBlock *InsertAE) - : LoadInst(Ptr, Name, isVolatile, Align, NotAtomic, CrossThread, InsertAE) { -} + : LoadInst(Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic, + CrossThread, InsertAE) {} LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, AtomicOrdering Order, @@ -1245,7 +1377,7 @@ LoadInst::LoadInst(Value *Ptr, const char *Name, Instruction *InsertBef) Load, Ptr, InsertBef) { setVolatile(false); setAlignment(0); - setAtomic(NotAtomic); + setAtomic(AtomicOrdering::NotAtomic); AssertOK(); if (Name && Name[0]) setName(Name); } @@ -1255,7 +1387,7 @@ LoadInst::LoadInst(Value *Ptr, const char *Name, BasicBlock *InsertAE) Load, Ptr, InsertAE) { setVolatile(false); setAlignment(0); - setAtomic(NotAtomic); + setAtomic(AtomicOrdering::NotAtomic); AssertOK(); if (Name && Name[0]) setName(Name); } @@ -1266,7 +1398,7 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const char *Name, bool isVolatile, assert(Ty == cast<PointerType>(Ptr->getType())->getElementType()); setVolatile(isVolatile); setAlignment(0); - setAtomic(NotAtomic); + setAtomic(AtomicOrdering::NotAtomic); AssertOK(); if (Name && Name[0]) setName(Name); } @@ -1277,7 +1409,7 @@ LoadInst::LoadInst(Value *Ptr, const char *Name, bool isVolatile, Load, Ptr, InsertAE) { setVolatile(isVolatile); setAlignment(0); - setAtomic(NotAtomic); + setAtomic(AtomicOrdering::NotAtomic); AssertOK(); if (Name && Name[0]) setName(Name); } @@ -1322,13 +1454,13 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, Instruction *InsertBefore) - : StoreInst(val, addr, isVolatile, Align, NotAtomic, CrossThread, - InsertBefore) {} + : StoreInst(val, addr, isVolatile, Align, AtomicOrdering::NotAtomic, + CrossThread, InsertBefore) {} StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, BasicBlock *InsertAtEnd) - : StoreInst(val, addr, isVolatile, Align, NotAtomic, CrossThread, - InsertAtEnd) {} + : StoreInst(val, addr, isVolatile, Align, AtomicOrdering::NotAtomic, + CrossThread, InsertAtEnd) {} StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, AtomicOrdering Order, @@ -1396,13 +1528,15 @@ void AtomicCmpXchgInst::Init(Value *Ptr, Value *Cmp, Value *NewVal, assert(getOperand(2)->getType() == cast<PointerType>(getOperand(0)->getType())->getElementType() && "Ptr must be a pointer to NewVal type!"); - assert(SuccessOrdering != NotAtomic && + assert(SuccessOrdering != AtomicOrdering::NotAtomic && "AtomicCmpXchg instructions must be atomic!"); - assert(FailureOrdering != NotAtomic && + assert(FailureOrdering != AtomicOrdering::NotAtomic && "AtomicCmpXchg instructions must be atomic!"); - assert(SuccessOrdering >= FailureOrdering && - "AtomicCmpXchg success ordering must be at least as strong as fail"); - assert(FailureOrdering != Release && FailureOrdering != AcquireRelease && + assert(!isStrongerThan(FailureOrdering, SuccessOrdering) && + "AtomicCmpXchg failure argument shall be no stronger than the success " + "argument"); + assert(FailureOrdering != AtomicOrdering::Release && + FailureOrdering != AtomicOrdering::AcquireRelease && "AtomicCmpXchg failure ordering cannot include release semantics"); } @@ -1452,7 +1586,7 @@ void AtomicRMWInst::Init(BinOp Operation, Value *Ptr, Value *Val, assert(getOperand(1)->getType() == cast<PointerType>(getOperand(0)->getType())->getElementType() && "Ptr must be a pointer to Val type!"); - assert(Ordering != NotAtomic && + assert(Ordering != AtomicOrdering::NotAtomic && "AtomicRMW instructions must be atomic!"); } @@ -2099,7 +2233,7 @@ static inline bool isConstantAllOnes(const Value *V) { bool BinaryOperator::isNeg(const Value *V) { if (const BinaryOperator *Bop = dyn_cast<BinaryOperator>(V)) if (Bop->getOpcode() == Instruction::Sub) - if (Constant* C = dyn_cast<Constant>(Bop->getOperand(0))) + if (Constant *C = dyn_cast<Constant>(Bop->getOperand(0))) return C->isNegativeZeroValue(); return false; } @@ -2107,7 +2241,7 @@ bool BinaryOperator::isNeg(const Value *V) { bool BinaryOperator::isFNeg(const Value *V, bool IgnoreZeroSign) { if (const BinaryOperator *Bop = dyn_cast<BinaryOperator>(V)) if (Bop->getOpcode() == Instruction::FSub) - if (Constant* C = dyn_cast<Constant>(Bop->getOperand(0))) { + if (Constant *C = dyn_cast<Constant>(Bop->getOperand(0))) { if (!IgnoreZeroSign) IgnoreZeroSign = cast<Instruction>(V)->hasNoSignedZeros(); return !IgnoreZeroSign ? C->isNegativeZeroValue() : C->isZeroValue(); @@ -2167,62 +2301,6 @@ bool BinaryOperator::swapOperands() { return false; } -void BinaryOperator::setHasNoUnsignedWrap(bool b) { - cast<OverflowingBinaryOperator>(this)->setHasNoUnsignedWrap(b); -} - -void BinaryOperator::setHasNoSignedWrap(bool b) { - cast<OverflowingBinaryOperator>(this)->setHasNoSignedWrap(b); -} - -void BinaryOperator::setIsExact(bool b) { - cast<PossiblyExactOperator>(this)->setIsExact(b); -} - -bool BinaryOperator::hasNoUnsignedWrap() const { - return cast<OverflowingBinaryOperator>(this)->hasNoUnsignedWrap(); -} - -bool BinaryOperator::hasNoSignedWrap() const { - return cast<OverflowingBinaryOperator>(this)->hasNoSignedWrap(); -} - -bool BinaryOperator::isExact() const { - return cast<PossiblyExactOperator>(this)->isExact(); -} - -void BinaryOperator::copyIRFlags(const Value *V) { - // Copy the wrapping flags. - if (auto *OB = dyn_cast<OverflowingBinaryOperator>(V)) { - setHasNoSignedWrap(OB->hasNoSignedWrap()); - setHasNoUnsignedWrap(OB->hasNoUnsignedWrap()); - } - - // Copy the exact flag. - if (auto *PE = dyn_cast<PossiblyExactOperator>(V)) - setIsExact(PE->isExact()); - - // Copy the fast-math flags. - if (auto *FP = dyn_cast<FPMathOperator>(V)) - copyFastMathFlags(FP->getFastMathFlags()); -} - -void BinaryOperator::andIRFlags(const Value *V) { - if (auto *OB = dyn_cast<OverflowingBinaryOperator>(V)) { - setHasNoSignedWrap(hasNoSignedWrap() & OB->hasNoSignedWrap()); - setHasNoUnsignedWrap(hasNoUnsignedWrap() & OB->hasNoUnsignedWrap()); - } - - if (auto *PE = dyn_cast<PossiblyExactOperator>(V)) - setIsExact(isExact() & PE->isExact()); - - if (auto *FP = dyn_cast<FPMathOperator>(V)) { - FastMathFlags FM = getFastMathFlags(); - FM &= FP->getFastMathFlags(); - copyFastMathFlags(FM); - } -} - //===----------------------------------------------------------------------===// // FPMathOperator Class @@ -2267,8 +2345,8 @@ bool CastInst::isLosslessCast() const { return false; // Identity cast is always lossless - Type* SrcTy = getOperand(0)->getType(); - Type* DstTy = getType(); + Type *SrcTy = getOperand(0)->getType(); + Type *DstTy = getType(); if (SrcTy == DstTy) return true; @@ -3575,6 +3653,34 @@ bool CmpInst::isFalseWhenEqual(Predicate predicate) { } } +bool CmpInst::isImpliedTrueByMatchingCmp(Predicate Pred1, Predicate Pred2) { + // If the predicates match, then we know the first condition implies the + // second is true. + if (Pred1 == Pred2) + return true; + + switch (Pred1) { + default: + break; + case ICMP_EQ: + // A == B implies A >=u B, A <=u B, A >=s B, and A <=s B are true. + return Pred2 == ICMP_UGE || Pred2 == ICMP_ULE || Pred2 == ICMP_SGE || + Pred2 == ICMP_SLE; + case ICMP_UGT: // A >u B implies A != B and A >=u B are true. + return Pred2 == ICMP_NE || Pred2 == ICMP_UGE; + case ICMP_ULT: // A <u B implies A != B and A <=u B are true. + return Pred2 == ICMP_NE || Pred2 == ICMP_ULE; + case ICMP_SGT: // A >s B implies A != B and A >=s B are true. + return Pred2 == ICMP_NE || Pred2 == ICMP_SGE; + case ICMP_SLT: // A <s B implies A != B and A <=s B are true. + return Pred2 == ICMP_NE || Pred2 == ICMP_SLE; + } + return false; +} + +bool CmpInst::isImpliedFalseByMatchingCmp(Predicate Pred1, Predicate Pred2) { + return isImpliedTrueByMatchingCmp(Pred1, getInversePredicate(Pred2)); +} //===----------------------------------------------------------------------===// // SwitchInst Implementation @@ -3809,6 +3915,7 @@ AllocaInst *AllocaInst::cloneImpl() const { AllocaInst *Result = new AllocaInst(getAllocatedType(), (Value *)getOperand(0), getAlignment()); Result->setUsedWithInAlloca(isUsedWithInAlloca()); + Result->setSwiftError(isSwiftError()); return Result; } diff --git a/lib/IR/IntrinsicInst.cpp b/lib/IR/IntrinsicInst.cpp index b9b5a29091dff..3f747117b7282 100644 --- a/lib/IR/IntrinsicInst.cpp +++ b/lib/IR/IntrinsicInst.cpp @@ -25,31 +25,18 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Metadata.h" +#include "llvm/Support/raw_ostream.h" using namespace llvm; //===----------------------------------------------------------------------===// /// DbgInfoIntrinsic - This is the common base class for debug info intrinsics /// -static Value *CastOperand(Value *C) { - if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) - if (CE->isCast()) - return CE->getOperand(0); - return nullptr; -} - -Value *DbgInfoIntrinsic::StripCast(Value *C) { - if (Value *CO = CastOperand(C)) { - C = StripCast(CO); - } else if (GlobalVariable *GV = dyn_cast<GlobalVariable>(C)) { - if (GV->hasInitializer()) - if (Value *CO = CastOperand(GV->getInitializer())) - C = StripCast(CO); - } - return dyn_cast<GlobalVariable>(C); -} +Value *DbgInfoIntrinsic::getVariableLocation(bool AllowNullOp) const { + Value *Op = getArgOperand(0); + if (AllowNullOp && !Op) + return nullptr; -static Value *getValueImpl(Value *Op) { auto *MD = cast<MetadataAsValue>(Op)->getMetadata(); if (auto *V = dyn_cast<ValueAsMetadata>(MD)) return V->getValue(); @@ -59,23 +46,40 @@ static Value *getValueImpl(Value *Op) { return nullptr; } -//===----------------------------------------------------------------------===// -/// DbgDeclareInst - This represents the llvm.dbg.declare instruction. -/// - -Value *DbgDeclareInst::getAddress() const { - if (!getArgOperand(0)) - return nullptr; - - return getValueImpl(getArgOperand(0)); -} +int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable, + StringRef Name) { + assert(Name.startswith("llvm.")); -//===----------------------------------------------------------------------===// -/// DbgValueInst - This represents the llvm.dbg.value instruction. -/// + // Do successive binary searches of the dotted name components. For + // "llvm.gc.experimental.statepoint.p1i8.p1i32", we will find the range of + // intrinsics starting with "llvm.gc", then "llvm.gc.experimental", then + // "llvm.gc.experimental.statepoint", and then we will stop as the range is + // size 1. During the search, we can skip the prefix that we already know is + // identical. By using strncmp we consider names with differing suffixes to + // be part of the equal range. + size_t CmpStart = 0; + size_t CmpEnd = 4; // Skip the "llvm" component. + const char *const *Low = NameTable.begin(); + const char *const *High = NameTable.end(); + const char *const *LastLow = Low; + while (CmpEnd < Name.size() && High - Low > 0) { + CmpStart = CmpEnd; + CmpEnd = Name.find('.', CmpStart + 1); + CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd; + auto Cmp = [CmpStart, CmpEnd](const char *LHS, const char *RHS) { + return strncmp(LHS + CmpStart, RHS + CmpStart, CmpEnd - CmpStart) < 0; + }; + LastLow = Low; + std::tie(Low, High) = std::equal_range(Low, High, Name.data(), Cmp); + } + if (High - Low > 0) + LastLow = Low; -const Value *DbgValueInst::getValue() const { - return const_cast<DbgValueInst *>(this)->getValue(); + if (LastLow == NameTable.end()) + return -1; + StringRef NameFound = *LastLow; + if (Name == NameFound || + (Name.startswith(NameFound) && Name[NameFound.size()] == '.')) + return LastLow - NameTable.begin(); + return -1; } - -Value *DbgValueInst::getValue() { return getValueImpl(getArgOperand(0)); } diff --git a/lib/IR/LLVMContext.cpp b/lib/IR/LLVMContext.cpp index 48b53b0f532af..d27fcfb1b7a82 100644 --- a/lib/IR/LLVMContext.cpp +++ b/lib/IR/LLVMContext.cpp @@ -13,23 +13,24 @@ //===----------------------------------------------------------------------===// #include "llvm/IR/LLVMContext.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" #include "LLVMContextImpl.h" -#include "llvm/IR/Constants.h" -#include "llvm/IR/DebugLoc.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/DiagnosticPrinter.h" -#include "llvm/IR/Instruction.h" #include "llvm/IR/Metadata.h" -#include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/SourceMgr.h" -#include <cctype> -using namespace llvm; - -static ManagedStatic<LLVMContext> GlobalContext; +#include "llvm/IR/Module.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" +#include <cassert> +#include <cstdlib> +#include <string> +#include <utility> -LLVMContext& llvm::getGlobalContext() { - return *GlobalContext; -} +using namespace llvm; LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { // Create the fixed metadata kinds. This is done in the same order as the @@ -128,6 +129,15 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { assert(AlignID == MD_align && "align kind id drifted"); (void)AlignID; + // Create the 'llvm.loop' metadata kind. + unsigned LoopID = getMDKindID("llvm.loop"); + assert(LoopID == MD_loop && "llvm.loop kind id drifted"); + (void)LoopID; + + unsigned TypeID = getMDKindID("type"); + assert(TypeID == MD_type && "type kind id drifted"); + (void)TypeID; + auto *DeoptEntry = pImpl->getOrInsertBundleTag("deopt"); assert(DeoptEntry->second == LLVMContext::OB_deopt && "deopt operand bundle id drifted!"); @@ -137,7 +147,13 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { assert(FuncletEntry->second == LLVMContext::OB_funclet && "funclet operand bundle id drifted!"); (void)FuncletEntry; + + auto *GCTransitionEntry = pImpl->getOrInsertBundleTag("gc-transition"); + assert(GCTransitionEntry->second == LLVMContext::OB_gc_transition && + "gc-transition operand bundle id drifted!"); + (void)GCTransitionEntry; } + LLVMContext::~LLVMContext() { delete pImpl; } void LLVMContext::addModule(Module *M) { @@ -180,6 +196,13 @@ void LLVMContext::setDiagnosticHandler(DiagnosticHandlerTy DiagnosticHandler, pImpl->RespectDiagnosticFilters = RespectFilters; } +void LLVMContext::setDiagnosticHotnessRequested(bool Requested) { + pImpl->DiagnosticHotnessRequested = Requested; +} +bool LLVMContext::getDiagnosticHotnessRequested() const { + return pImpl->DiagnosticHotnessRequested; +} + LLVMContext::DiagnosticHandlerTy LLVMContext::getDiagnosticHandler() const { return pImpl->DiagnosticHandler; } @@ -213,31 +236,14 @@ static bool isDiagnosticEnabled(const DiagnosticInfo &DI) { // pattern, passed via one of the -pass-remarks* flags, matches the name of // the pass that is emitting the diagnostic. If there is no match, ignore the // diagnostic and return. - switch (DI.getKind()) { - case llvm::DK_OptimizationRemark: - if (!cast<DiagnosticInfoOptimizationRemark>(DI).isEnabled()) - return false; - break; - case llvm::DK_OptimizationRemarkMissed: - if (!cast<DiagnosticInfoOptimizationRemarkMissed>(DI).isEnabled()) - return false; - break; - case llvm::DK_OptimizationRemarkAnalysis: - if (!cast<DiagnosticInfoOptimizationRemarkAnalysis>(DI).isEnabled()) - return false; - break; - case llvm::DK_OptimizationRemarkAnalysisFPCommute: - if (!cast<DiagnosticInfoOptimizationRemarkAnalysisFPCommute>(DI) - .isEnabled()) - return false; - break; - default: - break; - } + if (auto *Remark = dyn_cast<DiagnosticInfoOptimizationBase>(&DI)) + return Remark->isEnabled(); + return true; } -static const char *getDiagnosticMessagePrefix(DiagnosticSeverity Severity) { +const char * +LLVMContext::getDiagnosticMessagePrefix(DiagnosticSeverity Severity) { switch (Severity) { case DS_Error: return "error"; @@ -314,9 +320,34 @@ void LLVMContext::setGC(const Function &Fn, std::string GCName) { } It->second = std::move(GCName); } + const std::string &LLVMContext::getGC(const Function &Fn) { return pImpl->GCNames[&Fn]; } + void LLVMContext::deleteGC(const Function &Fn) { pImpl->GCNames.erase(&Fn); } + +bool LLVMContext::shouldDiscardValueNames() const { + return pImpl->DiscardValueNames; +} + +bool LLVMContext::isODRUniquingDebugTypes() const { return !!pImpl->DITypeMap; } + +void LLVMContext::enableDebugTypeODRUniquing() { + if (pImpl->DITypeMap) + return; + + pImpl->DITypeMap.emplace(); +} + +void LLVMContext::disableDebugTypeODRUniquing() { pImpl->DITypeMap.reset(); } + +void LLVMContext::setDiscardValueNames(bool Discard) { + pImpl->DiscardValueNames = Discard; +} + +OptBisect &LLVMContext::getOptBisect() { + return pImpl->getOptBisect(); +} diff --git a/lib/IR/LLVMContextImpl.cpp b/lib/IR/LLVMContextImpl.cpp index 5239b4f7d84a6..b0b2c61bdf1f3 100644 --- a/lib/IR/LLVMContextImpl.cpp +++ b/lib/IR/LLVMContextImpl.cpp @@ -16,6 +16,8 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Module.h" +#include "llvm/IR/OptBisect.h" +#include "llvm/Support/ManagedStatic.h" #include <algorithm> using namespace llvm; @@ -43,31 +45,12 @@ LLVMContextImpl::LLVMContextImpl(LLVMContext &C) DiagnosticHandler = nullptr; DiagnosticContext = nullptr; RespectDiagnosticFilters = false; + DiagnosticHotnessRequested = false; YieldCallback = nullptr; YieldOpaqueHandle = nullptr; NamedStructTypesUniqueID = 0; } -namespace { -struct DropReferences { - // Takes the value_type of a ConstantUniqueMap's internal map, whose 'second' - // is a Constant*. - template <typename PairT> void operator()(const PairT &P) { - P.second->dropAllReferences(); - } -}; - -// Temporary - drops pair.first instead of second. -struct DropFirst { - // Takes the value_type of a ConstantUniqueMap's internal map, whose 'second' - // is a Constant*. - template<typename PairT> - void operator()(const PairT &P) { - P.first->dropAllReferences(); - } -}; -} - LLVMContextImpl::~LLVMContextImpl() { // NOTE: We need to delete the contents of OwnedModules, but Module's dtor // will call LLVMContextImpl::removeModule, thus invalidating iterators into @@ -99,14 +82,14 @@ LLVMContextImpl::~LLVMContextImpl() { #include "llvm/IR/Metadata.def" // Free the constants. - std::for_each(ExprConstants.map_begin(), ExprConstants.map_end(), - DropFirst()); - std::for_each(ArrayConstants.map_begin(), ArrayConstants.map_end(), - DropFirst()); - std::for_each(StructConstants.map_begin(), StructConstants.map_end(), - DropFirst()); - std::for_each(VectorConstants.map_begin(), VectorConstants.map_end(), - DropFirst()); + for (auto *I : ExprConstants) + I->dropAllReferences(); + for (auto *I : ArrayConstants) + I->dropAllReferences(); + for (auto *I : StructConstants) + I->dropAllReferences(); + for (auto *I : VectorConstants) + I->dropAllReferences(); ExprConstants.freeConstants(); ArrayConstants.freeConstants(); StructConstants.freeConstants(); @@ -117,10 +100,9 @@ LLVMContextImpl::~LLVMContextImpl() { InlineAsms.freeConstants(); DeleteContainerSeconds(IntConstants); DeleteContainerSeconds(FPConstants); - - for (StringMap<ConstantDataSequential*>::iterator I = CDSConstants.begin(), - E = CDSConstants.end(); I != E; ++I) - delete I->second; + + for (auto &CDSConstant : CDSConstants) + delete CDSConstant.second; CDSConstants.clear(); // Destroy attributes. @@ -158,9 +140,6 @@ LLVMContextImpl::~LLVMContextImpl() { // Destroy ValuesAsMetadata. for (auto &Pair : ValuesAsMetadata) delete Pair.second; - - // Destroy MDStrings. - MDStringCache.clear(); } void LLVMContextImpl::dropTriviallyDeadConstantArrays() { @@ -168,10 +147,8 @@ void LLVMContextImpl::dropTriviallyDeadConstantArrays() { do { Changed = false; - for (auto I = ArrayConstants.map_begin(), E = ArrayConstants.map_end(); - I != E; ) { - auto *C = I->first; - I++; + for (auto I = ArrayConstants.begin(), E = ArrayConstants.end(); I != E;) { + auto *C = *I++; if (C->use_empty()) { Changed = true; C->destroyConstant(); @@ -257,3 +234,19 @@ void GetElementPtrConstantExpr::anchor() { } void CompareConstantExpr::anchor() { } +/// Singleton instance of the OptBisect class. +/// +/// This singleton is accessed via the LLVMContext::getOptBisect() function. It +/// provides a mechanism to disable passes and individual optimizations at +/// compile time based on a command line option (-opt-bisect-limit) in order to +/// perform a bisecting search for optimization-related problems. +/// +/// Even if multiple LLVMContext objects are created, they will all return the +/// same instance of OptBisect in order to provide a single bisect count. Any +/// code that uses the OptBisect object should be serialized when bisection is +/// enabled in order to enable a consistent bisect count. +static ManagedStatic<OptBisect> OptBisector; + +OptBisect &LLVMContextImpl::getOptBisect() { + return *OptBisector; +} diff --git a/lib/IR/LLVMContextImpl.h b/lib/IR/LLVMContextImpl.h index d42047d4e7759..7820e2ab958d5 100644 --- a/lib/IR/LLVMContextImpl.h +++ b/lib/IR/LLVMContextImpl.h @@ -32,6 +32,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/Support/Dwarf.h" #include <vector> namespace llvm { @@ -211,6 +212,17 @@ public: template <class NodeTy> struct MDNodeKeyImpl; template <class NodeTy> struct MDNodeInfo; +/// Configuration point for MDNodeInfo::isEqual(). +template <class NodeTy> struct MDNodeSubsetEqualImpl { + typedef MDNodeKeyImpl<NodeTy> KeyTy; + static bool isSubsetEqual(const KeyTy &LHS, const NodeTy *RHS) { + return false; + } + static bool isSubsetEqual(const NodeTy *LHS, const NodeTy *RHS) { + return false; + } +}; + /// \brief DenseMapInfo for MDTuple. /// /// Note that we don't need the is-function-local bit, since that's implicit in @@ -255,14 +267,14 @@ template <> struct MDNodeKeyImpl<DILocation> { /// \brief DenseMapInfo for GenericDINode. template <> struct MDNodeKeyImpl<GenericDINode> : MDNodeOpsKey { unsigned Tag; - StringRef Header; - MDNodeKeyImpl(unsigned Tag, StringRef Header, ArrayRef<Metadata *> DwarfOps) + MDString *Header; + MDNodeKeyImpl(unsigned Tag, MDString *Header, ArrayRef<Metadata *> DwarfOps) : MDNodeOpsKey(DwarfOps), Tag(Tag), Header(Header) {} MDNodeKeyImpl(const GenericDINode *N) - : MDNodeOpsKey(N, 1), Tag(N->getTag()), Header(N->getHeader()) {} + : MDNodeOpsKey(N, 1), Tag(N->getTag()), Header(N->getRawHeader()) {} bool isKeyOf(const GenericDINode *RHS) const { - return Tag == RHS->getTag() && Header == RHS->getHeader() && + return Tag == RHS->getTag() && Header == RHS->getRawHeader() && compareOps(RHS, 1); } @@ -290,35 +302,35 @@ template <> struct MDNodeKeyImpl<DISubrange> { template <> struct MDNodeKeyImpl<DIEnumerator> { int64_t Value; - StringRef Name; + MDString *Name; - MDNodeKeyImpl(int64_t Value, StringRef Name) : Value(Value), Name(Name) {} + MDNodeKeyImpl(int64_t Value, MDString *Name) : Value(Value), Name(Name) {} MDNodeKeyImpl(const DIEnumerator *N) - : Value(N->getValue()), Name(N->getName()) {} + : Value(N->getValue()), Name(N->getRawName()) {} bool isKeyOf(const DIEnumerator *RHS) const { - return Value == RHS->getValue() && Name == RHS->getName(); + return Value == RHS->getValue() && Name == RHS->getRawName(); } unsigned getHashValue() const { return hash_combine(Value, Name); } }; template <> struct MDNodeKeyImpl<DIBasicType> { unsigned Tag; - StringRef Name; + MDString *Name; uint64_t SizeInBits; uint64_t AlignInBits; unsigned Encoding; - MDNodeKeyImpl(unsigned Tag, StringRef Name, uint64_t SizeInBits, + MDNodeKeyImpl(unsigned Tag, MDString *Name, uint64_t SizeInBits, uint64_t AlignInBits, unsigned Encoding) : Tag(Tag), Name(Name), SizeInBits(SizeInBits), AlignInBits(AlignInBits), Encoding(Encoding) {} MDNodeKeyImpl(const DIBasicType *N) - : Tag(N->getTag()), Name(N->getName()), SizeInBits(N->getSizeInBits()), + : Tag(N->getTag()), Name(N->getRawName()), SizeInBits(N->getSizeInBits()), AlignInBits(N->getAlignInBits()), Encoding(N->getEncoding()) {} bool isKeyOf(const DIBasicType *RHS) const { - return Tag == RHS->getTag() && Name == RHS->getName() && + return Tag == RHS->getTag() && Name == RHS->getRawName() && SizeInBits == RHS->getSizeInBits() && AlignInBits == RHS->getAlignInBits() && Encoding == RHS->getEncoding(); @@ -330,7 +342,7 @@ template <> struct MDNodeKeyImpl<DIBasicType> { template <> struct MDNodeKeyImpl<DIDerivedType> { unsigned Tag; - StringRef Name; + MDString *Name; Metadata *File; unsigned Line; Metadata *Scope; @@ -341,7 +353,7 @@ template <> struct MDNodeKeyImpl<DIDerivedType> { unsigned Flags; Metadata *ExtraData; - MDNodeKeyImpl(unsigned Tag, StringRef Name, Metadata *File, unsigned Line, + MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, Metadata *ExtraData) @@ -349,14 +361,14 @@ template <> struct MDNodeKeyImpl<DIDerivedType> { BaseType(BaseType), SizeInBits(SizeInBits), AlignInBits(AlignInBits), OffsetInBits(OffsetInBits), Flags(Flags), ExtraData(ExtraData) {} MDNodeKeyImpl(const DIDerivedType *N) - : Tag(N->getTag()), Name(N->getName()), File(N->getRawFile()), + : Tag(N->getTag()), Name(N->getRawName()), File(N->getRawFile()), Line(N->getLine()), Scope(N->getRawScope()), BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()), AlignInBits(N->getAlignInBits()), OffsetInBits(N->getOffsetInBits()), Flags(N->getFlags()), ExtraData(N->getRawExtraData()) {} bool isKeyOf(const DIDerivedType *RHS) const { - return Tag == RHS->getTag() && Name == RHS->getName() && + return Tag == RHS->getTag() && Name == RHS->getRawName() && File == RHS->getRawFile() && Line == RHS->getLine() && Scope == RHS->getRawScope() && BaseType == RHS->getRawBaseType() && SizeInBits == RHS->getSizeInBits() && @@ -365,14 +377,53 @@ template <> struct MDNodeKeyImpl<DIDerivedType> { ExtraData == RHS->getRawExtraData(); } unsigned getHashValue() const { - return hash_combine(Tag, Name, File, Line, Scope, BaseType, SizeInBits, - AlignInBits, OffsetInBits, Flags, ExtraData); + // If this is a member inside an ODR type, only hash the type and the name. + // Otherwise the hash will be stronger than + // MDNodeSubsetEqualImpl::isODRMember(). + if (Tag == dwarf::DW_TAG_member && Name) + if (auto *CT = dyn_cast_or_null<DICompositeType>(Scope)) + if (CT->getRawIdentifier()) + return hash_combine(Name, Scope); + + // Intentionally computes the hash on a subset of the operands for + // performance reason. The subset has to be significant enough to avoid + // collision "most of the time". There is no correctness issue in case of + // collision because of the full check above. + return hash_combine(Tag, Name, File, Line, Scope, BaseType, Flags); + } +}; + +template <> struct MDNodeSubsetEqualImpl<DIDerivedType> { + typedef MDNodeKeyImpl<DIDerivedType> KeyTy; + static bool isSubsetEqual(const KeyTy &LHS, const DIDerivedType *RHS) { + return isODRMember(LHS.Tag, LHS.Scope, LHS.Name, RHS); + } + static bool isSubsetEqual(const DIDerivedType *LHS, const DIDerivedType *RHS) { + return isODRMember(LHS->getTag(), LHS->getRawScope(), LHS->getRawName(), + RHS); + } + + /// Subprograms compare equal if they declare the same function in an ODR + /// type. + static bool isODRMember(unsigned Tag, const Metadata *Scope, + const MDString *Name, const DIDerivedType *RHS) { + // Check whether the LHS is eligible. + if (Tag != dwarf::DW_TAG_member || !Name) + return false; + + auto *CT = dyn_cast_or_null<DICompositeType>(Scope); + if (!CT || !CT->getRawIdentifier()) + return false; + + // Compare to the RHS. + return Tag == RHS->getTag() && Name == RHS->getRawName() && + Scope == RHS->getRawScope(); } }; template <> struct MDNodeKeyImpl<DICompositeType> { unsigned Tag; - StringRef Name; + MDString *Name; Metadata *File; unsigned Line; Metadata *Scope; @@ -385,31 +436,31 @@ template <> struct MDNodeKeyImpl<DICompositeType> { unsigned RuntimeLang; Metadata *VTableHolder; Metadata *TemplateParams; - StringRef Identifier; + MDString *Identifier; - MDNodeKeyImpl(unsigned Tag, StringRef Name, Metadata *File, unsigned Line, + MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *File, unsigned Line, Metadata *Scope, Metadata *BaseType, uint64_t SizeInBits, uint64_t AlignInBits, uint64_t OffsetInBits, unsigned Flags, Metadata *Elements, unsigned RuntimeLang, Metadata *VTableHolder, Metadata *TemplateParams, - StringRef Identifier) + MDString *Identifier) : Tag(Tag), Name(Name), File(File), Line(Line), Scope(Scope), BaseType(BaseType), SizeInBits(SizeInBits), AlignInBits(AlignInBits), OffsetInBits(OffsetInBits), Flags(Flags), Elements(Elements), RuntimeLang(RuntimeLang), VTableHolder(VTableHolder), TemplateParams(TemplateParams), Identifier(Identifier) {} MDNodeKeyImpl(const DICompositeType *N) - : Tag(N->getTag()), Name(N->getName()), File(N->getRawFile()), + : Tag(N->getTag()), Name(N->getRawName()), File(N->getRawFile()), Line(N->getLine()), Scope(N->getRawScope()), BaseType(N->getRawBaseType()), SizeInBits(N->getSizeInBits()), AlignInBits(N->getAlignInBits()), OffsetInBits(N->getOffsetInBits()), Flags(N->getFlags()), Elements(N->getRawElements()), RuntimeLang(N->getRuntimeLang()), VTableHolder(N->getRawVTableHolder()), TemplateParams(N->getRawTemplateParams()), - Identifier(N->getIdentifier()) {} + Identifier(N->getRawIdentifier()) {} bool isKeyOf(const DICompositeType *RHS) const { - return Tag == RHS->getTag() && Name == RHS->getName() && + return Tag == RHS->getTag() && Name == RHS->getRawName() && File == RHS->getRawFile() && Line == RHS->getLine() && Scope == RHS->getRawScope() && BaseType == RHS->getRawBaseType() && SizeInBits == RHS->getSizeInBits() && @@ -419,49 +470,55 @@ template <> struct MDNodeKeyImpl<DICompositeType> { RuntimeLang == RHS->getRuntimeLang() && VTableHolder == RHS->getRawVTableHolder() && TemplateParams == RHS->getRawTemplateParams() && - Identifier == RHS->getIdentifier(); + Identifier == RHS->getRawIdentifier(); } unsigned getHashValue() const { - return hash_combine(Tag, Name, File, Line, Scope, BaseType, SizeInBits, - AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, - VTableHolder, TemplateParams, Identifier); + // Intentionally computes the hash on a subset of the operands for + // performance reason. The subset has to be significant enough to avoid + // collision "most of the time". There is no correctness issue in case of + // collision because of the full check above. + return hash_combine(Name, File, Line, BaseType, Scope, Elements, + TemplateParams); } }; template <> struct MDNodeKeyImpl<DISubroutineType> { unsigned Flags; + uint8_t CC; Metadata *TypeArray; - MDNodeKeyImpl(int64_t Flags, Metadata *TypeArray) - : Flags(Flags), TypeArray(TypeArray) {} + MDNodeKeyImpl(unsigned Flags, uint8_t CC, Metadata *TypeArray) + : Flags(Flags), CC(CC), TypeArray(TypeArray) {} MDNodeKeyImpl(const DISubroutineType *N) - : Flags(N->getFlags()), TypeArray(N->getRawTypeArray()) {} + : Flags(N->getFlags()), CC(N->getCC()), TypeArray(N->getRawTypeArray()) {} bool isKeyOf(const DISubroutineType *RHS) const { - return Flags == RHS->getFlags() && TypeArray == RHS->getRawTypeArray(); + return Flags == RHS->getFlags() && CC == RHS->getCC() && + TypeArray == RHS->getRawTypeArray(); } - unsigned getHashValue() const { return hash_combine(Flags, TypeArray); } + unsigned getHashValue() const { return hash_combine(Flags, CC, TypeArray); } }; template <> struct MDNodeKeyImpl<DIFile> { - StringRef Filename; - StringRef Directory; + MDString *Filename; + MDString *Directory; - MDNodeKeyImpl(StringRef Filename, StringRef Directory) + MDNodeKeyImpl(MDString *Filename, MDString *Directory) : Filename(Filename), Directory(Directory) {} MDNodeKeyImpl(const DIFile *N) - : Filename(N->getFilename()), Directory(N->getDirectory()) {} + : Filename(N->getRawFilename()), Directory(N->getRawDirectory()) {} bool isKeyOf(const DIFile *RHS) const { - return Filename == RHS->getFilename() && Directory == RHS->getDirectory(); + return Filename == RHS->getRawFilename() && + Directory == RHS->getRawDirectory(); } unsigned getHashValue() const { return hash_combine(Filename, Directory); } }; template <> struct MDNodeKeyImpl<DISubprogram> { Metadata *Scope; - StringRef Name; - StringRef LinkageName; + MDString *Name; + MDString *LinkageName; Metadata *File; unsigned Line; Metadata *Type; @@ -471,57 +528,102 @@ template <> struct MDNodeKeyImpl<DISubprogram> { Metadata *ContainingType; unsigned Virtuality; unsigned VirtualIndex; + int ThisAdjustment; unsigned Flags; bool IsOptimized; + Metadata *Unit; Metadata *TemplateParams; Metadata *Declaration; Metadata *Variables; - MDNodeKeyImpl(Metadata *Scope, StringRef Name, StringRef LinkageName, + MDNodeKeyImpl(Metadata *Scope, MDString *Name, MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition, unsigned ScopeLine, Metadata *ContainingType, unsigned Virtuality, - unsigned VirtualIndex, unsigned Flags, bool IsOptimized, - Metadata *TemplateParams, Metadata *Declaration, - Metadata *Variables) + unsigned VirtualIndex, int ThisAdjustment, unsigned Flags, + bool IsOptimized, Metadata *Unit, Metadata *TemplateParams, + Metadata *Declaration, Metadata *Variables) : Scope(Scope), Name(Name), LinkageName(LinkageName), File(File), Line(Line), Type(Type), IsLocalToUnit(IsLocalToUnit), IsDefinition(IsDefinition), ScopeLine(ScopeLine), ContainingType(ContainingType), Virtuality(Virtuality), - VirtualIndex(VirtualIndex), Flags(Flags), IsOptimized(IsOptimized), + VirtualIndex(VirtualIndex), ThisAdjustment(ThisAdjustment), + Flags(Flags), IsOptimized(IsOptimized), Unit(Unit), TemplateParams(TemplateParams), Declaration(Declaration), Variables(Variables) {} MDNodeKeyImpl(const DISubprogram *N) - : Scope(N->getRawScope()), Name(N->getName()), - LinkageName(N->getLinkageName()), File(N->getRawFile()), + : Scope(N->getRawScope()), Name(N->getRawName()), + LinkageName(N->getRawLinkageName()), File(N->getRawFile()), Line(N->getLine()), Type(N->getRawType()), IsLocalToUnit(N->isLocalToUnit()), IsDefinition(N->isDefinition()), ScopeLine(N->getScopeLine()), ContainingType(N->getRawContainingType()), Virtuality(N->getVirtuality()), VirtualIndex(N->getVirtualIndex()), - Flags(N->getFlags()), IsOptimized(N->isOptimized()), + ThisAdjustment(N->getThisAdjustment()), Flags(N->getFlags()), + IsOptimized(N->isOptimized()), Unit(N->getRawUnit()), TemplateParams(N->getRawTemplateParams()), Declaration(N->getRawDeclaration()), Variables(N->getRawVariables()) {} bool isKeyOf(const DISubprogram *RHS) const { - return Scope == RHS->getRawScope() && Name == RHS->getName() && - LinkageName == RHS->getLinkageName() && File == RHS->getRawFile() && - Line == RHS->getLine() && Type == RHS->getRawType() && - IsLocalToUnit == RHS->isLocalToUnit() && + return Scope == RHS->getRawScope() && Name == RHS->getRawName() && + LinkageName == RHS->getRawLinkageName() && + File == RHS->getRawFile() && Line == RHS->getLine() && + Type == RHS->getRawType() && IsLocalToUnit == RHS->isLocalToUnit() && IsDefinition == RHS->isDefinition() && ScopeLine == RHS->getScopeLine() && ContainingType == RHS->getRawContainingType() && Virtuality == RHS->getVirtuality() && - VirtualIndex == RHS->getVirtualIndex() && Flags == RHS->getFlags() && - IsOptimized == RHS->isOptimized() && + VirtualIndex == RHS->getVirtualIndex() && + ThisAdjustment == RHS->getThisAdjustment() && + Flags == RHS->getFlags() && IsOptimized == RHS->isOptimized() && + Unit == RHS->getUnit() && TemplateParams == RHS->getRawTemplateParams() && Declaration == RHS->getRawDeclaration() && Variables == RHS->getRawVariables(); } unsigned getHashValue() const { - return hash_combine(Scope, Name, LinkageName, File, Line, Type, - IsLocalToUnit, IsDefinition, ScopeLine, ContainingType, - Virtuality, VirtualIndex, Flags, IsOptimized, - TemplateParams, Declaration, Variables); + // If this is a declaration inside an ODR type, only hash the type and the + // name. Otherwise the hash will be stronger than + // MDNodeSubsetEqualImpl::isDeclarationOfODRMember(). + if (!IsDefinition && LinkageName) + if (auto *CT = dyn_cast_or_null<DICompositeType>(Scope)) + if (CT->getRawIdentifier()) + return hash_combine(LinkageName, Scope); + + // Intentionally computes the hash on a subset of the operands for + // performance reason. The subset has to be significant enough to avoid + // collision "most of the time". There is no correctness issue in case of + // collision because of the full check above. + return hash_combine(Name, Scope, File, Type, Line); + } +}; + +template <> struct MDNodeSubsetEqualImpl<DISubprogram> { + typedef MDNodeKeyImpl<DISubprogram> KeyTy; + static bool isSubsetEqual(const KeyTy &LHS, const DISubprogram *RHS) { + return isDeclarationOfODRMember(LHS.IsDefinition, LHS.Scope, + LHS.LinkageName, RHS); + } + static bool isSubsetEqual(const DISubprogram *LHS, const DISubprogram *RHS) { + return isDeclarationOfODRMember(LHS->isDefinition(), LHS->getRawScope(), + LHS->getRawLinkageName(), RHS); + } + + /// Subprograms compare equal if they declare the same function in an ODR + /// type. + static bool isDeclarationOfODRMember(bool IsDefinition, const Metadata *Scope, + const MDString *LinkageName, + const DISubprogram *RHS) { + // Check whether the LHS is eligible. + if (IsDefinition || !Scope || !LinkageName) + return false; + + auto *CT = dyn_cast_or_null<DICompositeType>(Scope); + if (!CT || !CT->getRawIdentifier()) + return false; + + // Compare to the RHS. + return IsDefinition == RHS->isDefinition() && Scope == RHS->getRawScope() && + LinkageName == RHS->getRawLinkageName(); } }; @@ -569,18 +671,18 @@ template <> struct MDNodeKeyImpl<DILexicalBlockFile> { template <> struct MDNodeKeyImpl<DINamespace> { Metadata *Scope; Metadata *File; - StringRef Name; + MDString *Name; unsigned Line; - MDNodeKeyImpl(Metadata *Scope, Metadata *File, StringRef Name, unsigned Line) + MDNodeKeyImpl(Metadata *Scope, Metadata *File, MDString *Name, unsigned Line) : Scope(Scope), File(File), Name(Name), Line(Line) {} MDNodeKeyImpl(const DINamespace *N) - : Scope(N->getRawScope()), File(N->getRawFile()), Name(N->getName()), + : Scope(N->getRawScope()), File(N->getRawFile()), Name(N->getRawName()), Line(N->getLine()) {} bool isKeyOf(const DINamespace *RHS) const { return Scope == RHS->getRawScope() && File == RHS->getRawFile() && - Name == RHS->getName() && Line == RHS->getLine(); + Name == RHS->getRawName() && Line == RHS->getLine(); } unsigned getHashValue() const { return hash_combine(Scope, File, Name, Line); @@ -589,26 +691,24 @@ template <> struct MDNodeKeyImpl<DINamespace> { template <> struct MDNodeKeyImpl<DIModule> { Metadata *Scope; - StringRef Name; - StringRef ConfigurationMacros; - StringRef IncludePath; - StringRef ISysRoot; - MDNodeKeyImpl(Metadata *Scope, StringRef Name, - StringRef ConfigurationMacros, - StringRef IncludePath, - StringRef ISysRoot) - : Scope(Scope), Name(Name), ConfigurationMacros(ConfigurationMacros), - IncludePath(IncludePath), ISysRoot(ISysRoot) {} + MDString *Name; + MDString *ConfigurationMacros; + MDString *IncludePath; + MDString *ISysRoot; + MDNodeKeyImpl(Metadata *Scope, MDString *Name, MDString *ConfigurationMacros, + MDString *IncludePath, MDString *ISysRoot) + : Scope(Scope), Name(Name), ConfigurationMacros(ConfigurationMacros), + IncludePath(IncludePath), ISysRoot(ISysRoot) {} MDNodeKeyImpl(const DIModule *N) - : Scope(N->getRawScope()), Name(N->getName()), - ConfigurationMacros(N->getConfigurationMacros()), - IncludePath(N->getIncludePath()), ISysRoot(N->getISysRoot()) {} + : Scope(N->getRawScope()), Name(N->getRawName()), + ConfigurationMacros(N->getRawConfigurationMacros()), + IncludePath(N->getRawIncludePath()), ISysRoot(N->getRawISysRoot()) {} bool isKeyOf(const DIModule *RHS) const { - return Scope == RHS->getRawScope() && Name == RHS->getName() && - ConfigurationMacros == RHS->getConfigurationMacros() && - IncludePath == RHS->getIncludePath() && - ISysRoot == RHS->getISysRoot(); + return Scope == RHS->getRawScope() && Name == RHS->getRawName() && + ConfigurationMacros == RHS->getRawConfigurationMacros() && + IncludePath == RHS->getRawIncludePath() && + ISysRoot == RHS->getRawISysRoot(); } unsigned getHashValue() const { return hash_combine(Scope, Name, @@ -617,33 +717,33 @@ template <> struct MDNodeKeyImpl<DIModule> { }; template <> struct MDNodeKeyImpl<DITemplateTypeParameter> { - StringRef Name; + MDString *Name; Metadata *Type; - MDNodeKeyImpl(StringRef Name, Metadata *Type) : Name(Name), Type(Type) {} + MDNodeKeyImpl(MDString *Name, Metadata *Type) : Name(Name), Type(Type) {} MDNodeKeyImpl(const DITemplateTypeParameter *N) - : Name(N->getName()), Type(N->getRawType()) {} + : Name(N->getRawName()), Type(N->getRawType()) {} bool isKeyOf(const DITemplateTypeParameter *RHS) const { - return Name == RHS->getName() && Type == RHS->getRawType(); + return Name == RHS->getRawName() && Type == RHS->getRawType(); } unsigned getHashValue() const { return hash_combine(Name, Type); } }; template <> struct MDNodeKeyImpl<DITemplateValueParameter> { unsigned Tag; - StringRef Name; + MDString *Name; Metadata *Type; Metadata *Value; - MDNodeKeyImpl(unsigned Tag, StringRef Name, Metadata *Type, Metadata *Value) + MDNodeKeyImpl(unsigned Tag, MDString *Name, Metadata *Type, Metadata *Value) : Tag(Tag), Name(Name), Type(Type), Value(Value) {} MDNodeKeyImpl(const DITemplateValueParameter *N) - : Tag(N->getTag()), Name(N->getName()), Type(N->getRawType()), + : Tag(N->getTag()), Name(N->getRawName()), Type(N->getRawType()), Value(N->getValue()) {} bool isKeyOf(const DITemplateValueParameter *RHS) const { - return Tag == RHS->getTag() && Name == RHS->getName() && + return Tag == RHS->getTag() && Name == RHS->getRawName() && Type == RHS->getRawType() && Value == RHS->getValue(); } unsigned getHashValue() const { return hash_combine(Tag, Name, Type, Value); } @@ -651,8 +751,8 @@ template <> struct MDNodeKeyImpl<DITemplateValueParameter> { template <> struct MDNodeKeyImpl<DIGlobalVariable> { Metadata *Scope; - StringRef Name; - StringRef LinkageName; + MDString *Name; + MDString *LinkageName; Metadata *File; unsigned Line; Metadata *Type; @@ -661,7 +761,7 @@ template <> struct MDNodeKeyImpl<DIGlobalVariable> { Metadata *Variable; Metadata *StaticDataMemberDeclaration; - MDNodeKeyImpl(Metadata *Scope, StringRef Name, StringRef LinkageName, + MDNodeKeyImpl(Metadata *Scope, MDString *Name, MDString *LinkageName, Metadata *File, unsigned Line, Metadata *Type, bool IsLocalToUnit, bool IsDefinition, Metadata *Variable, Metadata *StaticDataMemberDeclaration) @@ -670,18 +770,18 @@ template <> struct MDNodeKeyImpl<DIGlobalVariable> { IsDefinition(IsDefinition), Variable(Variable), StaticDataMemberDeclaration(StaticDataMemberDeclaration) {} MDNodeKeyImpl(const DIGlobalVariable *N) - : Scope(N->getRawScope()), Name(N->getName()), - LinkageName(N->getLinkageName()), File(N->getRawFile()), + : Scope(N->getRawScope()), Name(N->getRawName()), + LinkageName(N->getRawLinkageName()), File(N->getRawFile()), Line(N->getLine()), Type(N->getRawType()), IsLocalToUnit(N->isLocalToUnit()), IsDefinition(N->isDefinition()), Variable(N->getRawVariable()), StaticDataMemberDeclaration(N->getRawStaticDataMemberDeclaration()) {} bool isKeyOf(const DIGlobalVariable *RHS) const { - return Scope == RHS->getRawScope() && Name == RHS->getName() && - LinkageName == RHS->getLinkageName() && File == RHS->getRawFile() && - Line == RHS->getLine() && Type == RHS->getRawType() && - IsLocalToUnit == RHS->isLocalToUnit() && + return Scope == RHS->getRawScope() && Name == RHS->getRawName() && + LinkageName == RHS->getRawLinkageName() && + File == RHS->getRawFile() && Line == RHS->getLine() && + Type == RHS->getRawType() && IsLocalToUnit == RHS->isLocalToUnit() && IsDefinition == RHS->isDefinition() && Variable == RHS->getRawVariable() && StaticDataMemberDeclaration == @@ -696,24 +796,24 @@ template <> struct MDNodeKeyImpl<DIGlobalVariable> { template <> struct MDNodeKeyImpl<DILocalVariable> { Metadata *Scope; - StringRef Name; + MDString *Name; Metadata *File; unsigned Line; Metadata *Type; unsigned Arg; unsigned Flags; - MDNodeKeyImpl(Metadata *Scope, StringRef Name, Metadata *File, unsigned Line, + MDNodeKeyImpl(Metadata *Scope, MDString *Name, Metadata *File, unsigned Line, Metadata *Type, unsigned Arg, unsigned Flags) : Scope(Scope), Name(Name), File(File), Line(Line), Type(Type), Arg(Arg), Flags(Flags) {} MDNodeKeyImpl(const DILocalVariable *N) - : Scope(N->getRawScope()), Name(N->getName()), File(N->getRawFile()), + : Scope(N->getRawScope()), Name(N->getRawName()), File(N->getRawFile()), Line(N->getLine()), Type(N->getRawType()), Arg(N->getArg()), Flags(N->getFlags()) {} bool isKeyOf(const DILocalVariable *RHS) const { - return Scope == RHS->getRawScope() && Name == RHS->getName() && + return Scope == RHS->getRawScope() && Name == RHS->getRawName() && File == RHS->getRawFile() && Line == RHS->getLine() && Type == RHS->getRawType() && Arg == RHS->getArg() && Flags == RHS->getFlags(); @@ -738,28 +838,28 @@ template <> struct MDNodeKeyImpl<DIExpression> { }; template <> struct MDNodeKeyImpl<DIObjCProperty> { - StringRef Name; + MDString *Name; Metadata *File; unsigned Line; - StringRef GetterName; - StringRef SetterName; + MDString *GetterName; + MDString *SetterName; unsigned Attributes; Metadata *Type; - MDNodeKeyImpl(StringRef Name, Metadata *File, unsigned Line, - StringRef GetterName, StringRef SetterName, unsigned Attributes, + MDNodeKeyImpl(MDString *Name, Metadata *File, unsigned Line, + MDString *GetterName, MDString *SetterName, unsigned Attributes, Metadata *Type) : Name(Name), File(File), Line(Line), GetterName(GetterName), SetterName(SetterName), Attributes(Attributes), Type(Type) {} MDNodeKeyImpl(const DIObjCProperty *N) - : Name(N->getName()), File(N->getRawFile()), Line(N->getLine()), - GetterName(N->getGetterName()), SetterName(N->getSetterName()), + : Name(N->getRawName()), File(N->getRawFile()), Line(N->getLine()), + GetterName(N->getRawGetterName()), SetterName(N->getRawSetterName()), Attributes(N->getAttributes()), Type(N->getRawType()) {} bool isKeyOf(const DIObjCProperty *RHS) const { - return Name == RHS->getName() && File == RHS->getRawFile() && - Line == RHS->getLine() && GetterName == RHS->getGetterName() && - SetterName == RHS->getSetterName() && + return Name == RHS->getRawName() && File == RHS->getRawFile() && + Line == RHS->getLine() && GetterName == RHS->getRawGetterName() && + SetterName == RHS->getRawSetterName() && Attributes == RHS->getAttributes() && Type == RHS->getRawType(); } unsigned getHashValue() const { @@ -773,19 +873,19 @@ template <> struct MDNodeKeyImpl<DIImportedEntity> { Metadata *Scope; Metadata *Entity; unsigned Line; - StringRef Name; + MDString *Name; MDNodeKeyImpl(unsigned Tag, Metadata *Scope, Metadata *Entity, unsigned Line, - StringRef Name) + MDString *Name) : Tag(Tag), Scope(Scope), Entity(Entity), Line(Line), Name(Name) {} MDNodeKeyImpl(const DIImportedEntity *N) : Tag(N->getTag()), Scope(N->getRawScope()), Entity(N->getRawEntity()), - Line(N->getLine()), Name(N->getName()) {} + Line(N->getLine()), Name(N->getRawName()) {} bool isKeyOf(const DIImportedEntity *RHS) const { return Tag == RHS->getTag() && Scope == RHS->getRawScope() && Entity == RHS->getRawEntity() && Line == RHS->getLine() && - Name == RHS->getName(); + Name == RHS->getRawName(); } unsigned getHashValue() const { return hash_combine(Tag, Scope, Entity, Line, Name); @@ -795,18 +895,18 @@ template <> struct MDNodeKeyImpl<DIImportedEntity> { template <> struct MDNodeKeyImpl<DIMacro> { unsigned MIType; unsigned Line; - StringRef Name; - StringRef Value; + MDString *Name; + MDString *Value; - MDNodeKeyImpl(unsigned MIType, unsigned Line, StringRef Name, StringRef Value) + MDNodeKeyImpl(unsigned MIType, unsigned Line, MDString *Name, MDString *Value) : MIType(MIType), Line(Line), Name(Name), Value(Value) {} MDNodeKeyImpl(const DIMacro *N) - : MIType(N->getMacinfoType()), Line(N->getLine()), Name(N->getName()), - Value(N->getValue()) {} + : MIType(N->getMacinfoType()), Line(N->getLine()), Name(N->getRawName()), + Value(N->getRawValue()) {} bool isKeyOf(const DIMacro *RHS) const { return MIType == RHS->getMacinfoType() && Line == RHS->getLine() && - Name == RHS->getName() && Value == RHS->getValue(); + Name == RHS->getRawName() && Value == RHS->getRawValue(); } unsigned getHashValue() const { return hash_combine(MIType, Line, Name, Value); @@ -838,6 +938,7 @@ template <> struct MDNodeKeyImpl<DIMacroFile> { /// \brief DenseMapInfo for MDNode subclasses. template <class NodeTy> struct MDNodeInfo { typedef MDNodeKeyImpl<NodeTy> KeyTy; + typedef MDNodeSubsetEqualImpl<NodeTy> SubsetEqualTy; static inline NodeTy *getEmptyKey() { return DenseMapInfo<NodeTy *>::getEmptyKey(); } @@ -851,10 +952,14 @@ template <class NodeTy> struct MDNodeInfo { static bool isEqual(const KeyTy &LHS, const NodeTy *RHS) { if (RHS == getEmptyKey() || RHS == getTombstoneKey()) return false; - return LHS.isKeyOf(RHS); + return SubsetEqualTy::isSubsetEqual(LHS, RHS) || LHS.isKeyOf(RHS); } static bool isEqual(const NodeTy *LHS, const NodeTy *RHS) { - return LHS == RHS; + if (LHS == RHS) + return true; + if (RHS == getEmptyKey() || RHS == getTombstoneKey()) + return false; + return SubsetEqualTy::isSubsetEqual(LHS, RHS); } }; @@ -899,6 +1004,33 @@ public: } }; +/// Multimap-like storage for metadata attachments for globals. This differs +/// from MDAttachmentMap in that it allows multiple attachments per metadata +/// kind. +class MDGlobalAttachmentMap { + struct Attachment { + unsigned MDKind; + TrackingMDNodeRef Node; + }; + SmallVector<Attachment, 1> Attachments; + +public: + bool empty() const { return Attachments.empty(); } + + /// Appends all attachments with the given ID to \c Result in insertion order. + /// If the global has no attachments with the given ID, or if ID is invalid, + /// leaves Result unchanged. + void get(unsigned ID, SmallVectorImpl<MDNode *> &Result); + + void insert(unsigned ID, MDNode &MD); + void erase(unsigned ID); + + /// Appends all attachments for the global to \c Result, sorting by attachment + /// ID. Attachments with the same ID appear in insertion order. This function + /// does \em not clear \c Result. + void getAll(SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const; +}; + class LLVMContextImpl { public: /// OwnedModules - The set of modules instantiated in this context, and which @@ -911,6 +1043,7 @@ public: LLVMContext::DiagnosticHandlerTy DiagnosticHandler; void *DiagnosticContext; bool RespectDiagnosticFilters; + bool DiagnosticHotnessRequested; LLVMContext::YieldCallbackTy YieldCallback; void *YieldOpaqueHandle; @@ -925,7 +1058,7 @@ public: FoldingSet<AttributeSetImpl> AttrsLists; FoldingSet<AttributeSetNode> AttrsSetNodes; - StringMap<MDString> MDStringCache; + StringMap<MDString, BumpPtrAllocator> MDStringCache; DenseMap<Value *, ValueAsMetadata *> ValuesAsMetadata; DenseMap<Metadata *, MetadataAsValue *> MetadataAsValues; @@ -935,11 +1068,14 @@ public: DenseSet<CLASS *, CLASS##Info> CLASS##s; #include "llvm/IR/Metadata.def" + // Optional map for looking up composite types by identifier. + Optional<DenseMap<const MDString *, DICompositeType *>> DITypeMap; + // MDNodes may be uniqued or not uniqued. When they're not uniqued, they // aren't in the MDNodeSet, but they're still shared between objects, so no - // one object can destroy them. This set allows us to at least destroy them - // on Context destruction. - SmallPtrSet<MDNode *, 1> DistinctMDNodes; + // one object can destroy them. Keep track of them here so we can delete + // them on context teardown. + std::vector<MDNode *> DistinctMDNodes; DenseMap<Type*, ConstantAggregateZero*> CAZConstants; @@ -1006,8 +1142,8 @@ public: /// Collection of per-instruction metadata used in this context. DenseMap<const Instruction *, MDAttachmentMap> InstructionMetadata; - /// Collection of per-function metadata used in this context. - DenseMap<const Function *, MDAttachmentMap> FunctionMetadata; + /// Collection of per-GlobalObject metadata used in this context. + DenseMap<const GlobalObject *, MDGlobalAttachmentMap> GlobalObjectMetadata; /// DiscriminatorTable - This table maps file:line locations to an /// integer representing the next DWARF path discriminator to assign to @@ -1034,11 +1170,19 @@ public: /// clients which do use GC. DenseMap<const Function*, std::string> GCNames; + /// Flag to indicate if Value (other than GlobalValue) retains their name or + /// not. + bool DiscardValueNames = false; + LLVMContextImpl(LLVMContext &C); ~LLVMContextImpl(); /// Destroy the ConstantArrays if they are not used. void dropTriviallyDeadConstantArrays(); + + /// \brief Access the object which manages optimization bisection for failure + /// analysis. + OptBisect &getOptBisect(); }; } diff --git a/lib/IR/LegacyPassManager.cpp b/lib/IR/LegacyPassManager.cpp index 63d89f21b350e..8f71d822d2719 100644 --- a/lib/IR/LegacyPassManager.cpp +++ b/lib/IR/LegacyPassManager.cpp @@ -325,14 +325,6 @@ public: using llvm::Pass::doInitialization; using llvm::Pass::doFinalization; - /// doInitialization - Run all of the initializers for the module passes. - /// - bool doInitialization(); - - /// doFinalization - Run all of the finalizers for the module passes. - /// - bool doFinalization(); - /// Pass Manager itself does not invalidate any analysis info. void getAnalysisUsage(AnalysisUsage &Info) const override { Info.setPreservesAll(); @@ -423,14 +415,6 @@ public: using llvm::Pass::doInitialization; using llvm::Pass::doFinalization; - /// doInitialization - Run all of the initializers for the module passes. - /// - bool doInitialization(); - - /// doFinalization - Run all of the finalizers for the module passes. - /// - bool doFinalization(); - /// Pass Manager itself does not invalidate any analysis info. void getAnalysisUsage(AnalysisUsage &Info) const override { Info.setPreservesAll(); @@ -531,9 +515,8 @@ PMTopLevelManager::setLastUser(ArrayRef<Pass*> AnalysisPasses, Pass *P) { const AnalysisUsage::VectorType &IDs = AnUsage->getRequiredTransitiveSet(); SmallVector<Pass *, 12> LastUses; SmallVector<Pass *, 12> LastPMUses; - for (AnalysisUsage::VectorType::const_iterator I = IDs.begin(), - E = IDs.end(); I != E; ++I) { - Pass *AnalysisPass = findAnalysisPass(*I); + for (AnalysisID ID : IDs) { + Pass *AnalysisPass = findAnalysisPass(ID); assert(AnalysisPass && "Expected analysis pass to exist."); AnalysisResolver *AR = AnalysisPass->getResolver(); assert(AR && "Expected analysis resolver to exist."); @@ -791,29 +774,24 @@ void PMTopLevelManager::dumpArguments() const { return; dbgs() << "Pass Arguments: "; - for (SmallVectorImpl<ImmutablePass *>::const_iterator I = - ImmutablePasses.begin(), E = ImmutablePasses.end(); I != E; ++I) - if (const PassInfo *PI = findAnalysisPassInfo((*I)->getPassID())) { + for (ImmutablePass *P : ImmutablePasses) + if (const PassInfo *PI = findAnalysisPassInfo(P->getPassID())) { assert(PI && "Expected all immutable passes to be initialized"); if (!PI->isAnalysisGroup()) dbgs() << " -" << PI->getPassArgument(); } - for (SmallVectorImpl<PMDataManager *>::const_iterator I = - PassManagers.begin(), E = PassManagers.end(); I != E; ++I) - (*I)->dumpPassArguments(); + for (PMDataManager *PM : PassManagers) + PM->dumpPassArguments(); dbgs() << "\n"; } void PMTopLevelManager::initializeAllAnalysisInfo() { - for (SmallVectorImpl<PMDataManager *>::iterator I = PassManagers.begin(), - E = PassManagers.end(); I != E; ++I) - (*I)->initializeAnalysisInfo(); + for (PMDataManager *PM : PassManagers) + PM->initializeAnalysisInfo(); // Initailize other pass managers - for (SmallVectorImpl<PMDataManager *>::iterator - I = IndirectPassManagers.begin(), E = IndirectPassManagers.end(); - I != E; ++I) - (*I)->initializeAnalysisInfo(); + for (PMDataManager *IPM : IndirectPassManagers) + IPM->initializeAnalysisInfo(); for (DenseMap<Pass *, Pass *>::iterator DMI = LastUser.begin(), DME = LastUser.end(); DMI != DME; ++DMI) { @@ -824,13 +802,11 @@ void PMTopLevelManager::initializeAllAnalysisInfo() { /// Destructor PMTopLevelManager::~PMTopLevelManager() { - for (SmallVectorImpl<PMDataManager *>::iterator I = PassManagers.begin(), - E = PassManagers.end(); I != E; ++I) - delete *I; + for (PMDataManager *PM : PassManagers) + delete PM; - for (SmallVectorImpl<ImmutablePass *>::iterator - I = ImmutablePasses.begin(), E = ImmutablePasses.end(); I != E; ++I) - delete *I; + for (ImmutablePass *P : ImmutablePasses) + delete P; } //===----------------------------------------------------------------------===// @@ -1827,7 +1803,7 @@ void PMStack::push(PMDataManager *PM) { } // Dump content of the pass manager stack. -void PMStack::dump() const { +LLVM_DUMP_METHOD void PMStack::dump() const { for (PMDataManager *Manager : S) dbgs() << Manager->getAsPass()->getPassName() << ' '; diff --git a/lib/IR/MDBuilder.cpp b/lib/IR/MDBuilder.cpp index 4ce3ea2e9c041..a5a4cd06db09e 100644 --- a/lib/IR/MDBuilder.cpp +++ b/lib/IR/MDBuilder.cpp @@ -40,7 +40,7 @@ MDNode *MDBuilder::createBranchWeights(uint32_t TrueWeight, } MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights) { - assert(Weights.size() >= 2 && "Need at least two branch weights!"); + assert(Weights.size() >= 1 && "Need at least one branch weights!"); SmallVector<Metadata *, 4> Vals(Weights.size() + 1); Vals[0] = createString("branch_weights"); diff --git a/lib/IR/Makefile b/lib/IR/Makefile deleted file mode 100644 index 329cd6636e94d..0000000000000 --- a/lib/IR/Makefile +++ /dev/null @@ -1,61 +0,0 @@ -##===- lib/IR/Makefile -------------------------------------*- Makefile -*-===## -# -# The LLVM Compiler Infrastructure -# -# This file is distributed under the University of Illinois Open Source -# License. See LICENSE.TXT for details. -# -##===----------------------------------------------------------------------===## -LEVEL = ../.. -LIBRARYNAME = LLVMCore -BUILD_ARCHIVE = 1 - -BUILT_SOURCES = $(PROJ_OBJ_ROOT)/include/llvm/IR/Intrinsics.gen \ - $(PROJ_OBJ_ROOT)/include/llvm/IR/Attributes.inc \ - $(PROJ_OBJ_ROOT)/lib/IR/AttributesCompatFunc.inc - -include $(LEVEL)/Makefile.common - -GENFILE:=$(PROJ_OBJ_ROOT)/include/llvm/IR/Intrinsics.gen -ATTRINCFILE:=$(PROJ_OBJ_ROOT)/include/llvm/IR/Attributes.inc -ATTRCOMPATFUNCINCFILE:=$(PROJ_OBJ_ROOT)/lib/IR/AttributesCompatFunc.inc - -INTRINSICTD := $(PROJ_SRC_ROOT)/include/llvm/IR/Intrinsics.td -INTRINSICTDS := $(wildcard $(PROJ_SRC_ROOT)/include/llvm/IR/Intrinsics*.td) -ATTRIBUTESTD := $(PROJ_SRC_ROOT)/include/llvm/IR/Attributes.td -ATTRCOMPATFUNCTD := $(PROJ_SRC_ROOT)/lib/IR/AttributesCompatFunc.td - -$(ObjDir)/Intrinsics.gen.tmp: $(ObjDir)/.dir $(INTRINSICTDS) $(LLVM_TBLGEN) - $(Echo) Building Intrinsics.gen.tmp from Intrinsics.td - $(Verb) $(LLVMTableGen) $(call SYSPATH, $(INTRINSICTD)) -o $(call SYSPATH, $@) -gen-intrinsic - -$(GENFILE): $(ObjDir)/Intrinsics.gen.tmp $(PROJ_OBJ_ROOT)/include/llvm/IR/.dir - $(Verb) $(CMP) -s $@ $< || ( $(CP) $< $@ && \ - $(EchoCmd) Updated Intrinsics.gen because Intrinsics.gen.tmp \ - changed significantly. ) - -$(ObjDir)/Attributes.inc.tmp: $(ObjDir)/.dir $(ATTRIBUTESTD) $(LLVM_TBLGEN) - $(Echo) Building Attributes.inc.tmp from $(ATTRIBUTESTD) - $(Verb) $(LLVMTableGen) $(call SYSPATH, $(ATTRIBUTESTD)) -o $(call SYSPATH, $@) -gen-attrs - -$(ATTRINCFILE): $(ObjDir)/Attributes.inc.tmp $(PROJ_OBJ_ROOT)/include/llvm/IR/.dir - $(Verb) $(CMP) -s $@ $< || ( $(CP) $< $@ && \ - $(EchoCmd) Updated Attributes.inc because Attributes.inc.tmp \ - changed significantly. ) - -$(ObjDir)/AttributesCompatFunc.inc.tmp: $(ObjDir)/.dir $(ATTRCOMPATFUNCTD) $(LLVM_TBLGEN) - $(Echo) Building AttributesCompatFunc.inc.tmp from $(ATTRCOMPATFUNCTD) - $(Verb) $(LLVMTableGen) $(call SYSPATH, $(ATTRCOMPATFUNCTD)) -o $(call SYSPATH, $@) -gen-attrs - -$(ATTRCOMPATFUNCINCFILE): $(ObjDir)/AttributesCompatFunc.inc.tmp $(PROJ_OBJ_ROOT)/include/llvm/IR/.dir - $(Verb) $(CMP) -s $@ $< || ( $(CP) $< $@ && \ - $(EchoCmd) Updated AttributesCompatFunc.inc because AttributesCompatFunc.inc.tmp \ - changed significantly. ) - -install-local:: $(GENFILE) - $(Echo) Installing $(DESTDIR)$(PROJ_includedir)/llvm/IR/Intrinsics.gen - $(Verb) $(DataInstall) $(GENFILE) $(DESTDIR)$(PROJ_includedir)/llvm/IR/Intrinsics.gen - -install-local:: $(ATTRINCFILE) - $(Echo) Installing $(DESTDIR)$(PROJ_includedir)/llvm/IR/Attributes.inc - $(Verb) $(DataInstall) $(ATTRINCFILE) $(DESTDIR)$(PROJ_includedir)/llvm/IR/Attributes.inc diff --git a/lib/IR/Mangler.cpp b/lib/IR/Mangler.cpp index 016cb9eb68926..ddf024df8cccd 100644 --- a/lib/IR/Mangler.cpp +++ b/lib/IR/Mangler.cpp @@ -99,7 +99,7 @@ static void addByteCountSuffix(raw_ostream &OS, const Function *F, Ty = cast<PointerType>(Ty)->getElementType(); // Size should be aligned to pointer size. unsigned PtrSize = DL.getPointerSize(); - ArgWords += RoundUpToAlignment(DL.getTypeAllocSize(Ty), PtrSize); + ArgWords += alignTo(DL.getTypeAllocSize(Ty), PtrSize); } OS << '@' << ArgWords; diff --git a/lib/IR/Metadata.cpp b/lib/IR/Metadata.cpp index 9a9a5017841c0..5201c2ecce6ae 100644 --- a/lib/IR/Metadata.cpp +++ b/lib/IR/Metadata.cpp @@ -15,10 +15,8 @@ #include "LLVMContextImpl.h" #include "MetadataImpl.h" #include "SymbolTableListTraitsImpl.h" -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -39,7 +37,7 @@ MetadataAsValue::~MetadataAsValue() { untrack(); } -/// \brief Canonicalize metadata arguments to intrinsics. +/// Canonicalize metadata arguments to intrinsics. /// /// To support bitcode upgrades (and assembly semantic sugar) for \a /// MetadataAsValue, we need to canonicalize certain metadata. @@ -124,32 +122,44 @@ bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) { assert(Ref && "Expected live reference"); assert((Owner || *static_cast<Metadata **>(Ref) == &MD) && "Reference without owner must be direct"); - if (auto *R = ReplaceableMetadataImpl::get(MD)) { + if (auto *R = ReplaceableMetadataImpl::getOrCreate(MD)) { R->addRef(Ref, Owner); return true; } + if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) { + assert(!PH->Use && "Placeholders can only be used once"); + assert(!Owner && "Unexpected callback to owner"); + PH->Use = static_cast<Metadata **>(Ref); + return true; + } return false; } void MetadataTracking::untrack(void *Ref, Metadata &MD) { assert(Ref && "Expected live reference"); - if (auto *R = ReplaceableMetadataImpl::get(MD)) + if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) R->dropRef(Ref); + else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) + PH->Use = nullptr; } bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) { assert(Ref && "Expected live reference"); assert(New && "Expected live reference"); assert(Ref != New && "Expected change"); - if (auto *R = ReplaceableMetadataImpl::get(MD)) { + if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) { R->moveRef(Ref, New, MD); return true; } + assert(!isa<DistinctMDOperandPlaceholder>(MD) && + "Unexpected move of an MDOperand"); + assert(!isReplaceable(MD) && + "Expected un-replaceable metadata, since we didn't move a reference"); return false; } bool MetadataTracking::isReplaceable(const Metadata &MD) { - return ReplaceableMetadataImpl::get(const_cast<Metadata &>(MD)); + return ReplaceableMetadataImpl::isReplaceable(MD); } void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) { @@ -188,11 +198,6 @@ void ReplaceableMetadataImpl::moveRef(void *Ref, void *New, } void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) { - assert(!(MD && isa<MDNode>(MD) && cast<MDNode>(MD)->isTemporary()) && - "Expected non-temp node"); - assert(CanReplace && - "Attempted to replace Metadata marked for no replacement"); - if (UseMap.empty()) return; @@ -273,9 +278,21 @@ void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) { } } -ReplaceableMetadataImpl *ReplaceableMetadataImpl::get(Metadata &MD) { +ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) { + if (auto *N = dyn_cast<MDNode>(&MD)) + return N->isResolved() ? nullptr : N->Context.getOrCreateReplaceableUses(); + return dyn_cast<ValueAsMetadata>(&MD); +} + +ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) { if (auto *N = dyn_cast<MDNode>(&MD)) - return N->Context.getReplaceableUses(); + return N->isResolved() ? nullptr : N->Context.getReplaceableUses(); + return dyn_cast<ValueAsMetadata>(&MD); +} + +bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) { + if (auto *N = dyn_cast<MDNode>(&MD)) + return !N->isResolved(); return dyn_cast<ValueAsMetadata>(&MD); } @@ -399,17 +416,12 @@ void ValueAsMetadata::handleRAUW(Value *From, Value *To) { MDString *MDString::get(LLVMContext &Context, StringRef Str) { auto &Store = Context.pImpl->MDStringCache; - auto I = Store.find(Str); - if (I != Store.end()) - return &I->second; - - auto *Entry = - StringMapEntry<MDString>::Create(Str, Store.getAllocator(), MDString()); - bool WasInserted = Store.insert(Entry); - (void)WasInserted; - assert(WasInserted && "Expected entry to be inserted"); - Entry->second.Entry = Entry; - return &Entry->second; + auto I = Store.emplace_second(Str); + auto &MapEntry = I.first->getValue(); + if (!I.second) + return &MapEntry; + MapEntry.Entry = &*I.first; + return &MapEntry; } StringRef MDString::getString() const { @@ -433,7 +445,7 @@ void *MDNode::operator new(size_t Size, unsigned NumOps) { size_t OpSize = NumOps * sizeof(MDOperand); // uint64_t is the most aligned type we need support (ensured by static_assert // above) - OpSize = RoundUpToAlignment(OpSize, llvm::alignOf<uint64_t>()); + OpSize = alignTo(OpSize, llvm::alignOf<uint64_t>()); void *Ptr = reinterpret_cast<char *>(::operator new(OpSize + Size)) + OpSize; MDOperand *O = static_cast<MDOperand *>(Ptr); for (MDOperand *E = O - NumOps; O != E; --O) @@ -444,7 +456,7 @@ void *MDNode::operator new(size_t Size, unsigned NumOps) { void MDNode::operator delete(void *Mem) { MDNode *N = static_cast<MDNode *>(Mem); size_t OpSize = N->NumOperands * sizeof(MDOperand); - OpSize = RoundUpToAlignment(OpSize, llvm::alignOf<uint64_t>()); + OpSize = alignTo(OpSize, llvm::alignOf<uint64_t>()); MDOperand *O = static_cast<MDOperand *>(Mem); for (MDOperand *E = O - N->NumOperands; O != E; --O) @@ -462,16 +474,12 @@ MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage, for (Metadata *MD : Ops2) setOperand(Op++, MD); - if (isDistinct()) + if (!isUniqued()) return; - if (isUniqued()) - // Check whether any operands are unresolved, requiring re-uniquing. If - // not, don't support RAUW. - if (!countUnresolvedOperands()) - return; - - this->Context.makeReplaceable(make_unique<ReplaceableMetadataImpl>(Context)); + // Count the unresolved operands. If there are any, RAUW support will be + // added lazily on first reference. + countUnresolvedOperands(); } TempMDNode MDNode::clone() const { @@ -491,10 +499,10 @@ static bool isOperandUnresolved(Metadata *Op) { return false; } -unsigned MDNode::countUnresolvedOperands() { +void MDNode::countUnresolvedOperands() { assert(NumUnresolved == 0 && "Expected unresolved ops to be uncounted"); - NumUnresolved = std::count_if(op_begin(), op_end(), isOperandUnresolved); - return NumUnresolved; + assert(isUniqued() && "Expected this to be uniqued"); + NumUnresolved = count_if(operands(), isOperandUnresolved); } void MDNode::makeUniqued() { @@ -507,8 +515,11 @@ void MDNode::makeUniqued() { // Make this 'uniqued'. Storage = Uniqued; - if (!countUnresolvedOperands()) - resolve(); + countUnresolvedOperands(); + if (!NumUnresolved) { + dropReplaceableUses(); + assert(isResolved() && "Expected this to be resolved"); + } assert(isUniqued() && "Expected this to be uniqued"); } @@ -517,9 +528,8 @@ void MDNode::makeDistinct() { assert(isTemporary() && "Expected this to be temporary"); assert(!isResolved() && "Expected this to be unresolved"); - // Pretend to be uniqued, resolve the node, and then store in distinct table. - Storage = Uniqued; - resolve(); + // Drop RAUW support and store as a distinct node. + dropReplaceableUses(); storeDistinctInContext(); assert(isDistinct() && "Expected this to be distinct"); @@ -530,16 +540,22 @@ void MDNode::resolve() { assert(isUniqued() && "Expected this to be uniqued"); assert(!isResolved() && "Expected this to be unresolved"); - // Move the map, so that this immediately looks resolved. - auto Uses = Context.takeReplaceableUses(); NumUnresolved = 0; + dropReplaceableUses(); + assert(isResolved() && "Expected this to be resolved"); +} - // Drop RAUW support. - Uses->resolveAllUses(); +void MDNode::dropReplaceableUses() { + assert(!NumUnresolved && "Unexpected unresolved operand"); + + // Drop any RAUW support. + if (Context.hasReplaceableUses()) + Context.takeReplaceableUses()->resolveAllUses(); } void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) { + assert(isUniqued() && "Expected this to be uniqued"); assert(NumUnresolved != 0 && "Expected unresolved operands"); // Check if an operand was resolved. @@ -552,12 +568,20 @@ void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) { } void MDNode::decrementUnresolvedOperandCount() { - if (!--NumUnresolved) - // Last unresolved operand has just been resolved. - resolve(); + assert(!isResolved() && "Expected this to be unresolved"); + if (isTemporary()) + return; + + assert(isUniqued() && "Expected this to be uniqued"); + if (--NumUnresolved) + return; + + // Last unresolved operand has just been resolved. + dropReplaceableUses(); + assert(isResolved() && "Expected this to become resolved"); } -void MDNode::resolveRecursivelyImpl(bool AllowTemps) { +void MDNode::resolveCycles() { if (isResolved()) return; @@ -570,8 +594,6 @@ void MDNode::resolveRecursivelyImpl(bool AllowTemps) { if (!N) continue; - if (N->isTemporary() && AllowTemps) - continue; assert(!N->isTemporary() && "Expected all forward declarations to be resolved"); if (!N->isResolved()) @@ -631,7 +653,7 @@ void MDTuple::recalculateHash() { void MDNode::dropAllReferences() { for (unsigned I = 0, E = NumOperands; I != E; ++I) setOperand(I, nullptr); - if (!isResolved()) { + if (Context.hasReplaceableUses()) { Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false); (void)Context.takeReplaceableUses(); } @@ -677,7 +699,8 @@ void MDNode::handleChangedOperand(void *Ref, Metadata *New) { // dropAllReferences(), but we still need the use-list). for (unsigned O = 0, E = getNumOperands(); O != E; ++O) setOperand(O, nullptr); - Context.getReplaceableUses()->replaceAllUsesWith(Uniqued); + if (Context.hasReplaceableUses()) + Context.getReplaceableUses()->replaceAllUsesWith(Uniqued); deleteAsSubclass(); return; } @@ -775,8 +798,10 @@ void MDNode::deleteTemporary(MDNode *N) { } void MDNode::storeDistinctInContext() { - assert(isResolved() && "Expected resolved nodes"); + assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses"); + assert(!NumUnresolved && "Unexpected unresolved nodes"); Storage = Distinct; + assert(isResolved() && "Expected this to be resolved"); // Reset the hash. switch (getMetadataID()) { @@ -791,7 +816,7 @@ void MDNode::storeDistinctInContext() { #include "llvm/IR/Metadata.def" } - getContext().pImpl->DistinctMDNodes.insert(this); + getContext().pImpl->DistinctMDNodes.push_back(this); } void MDNode::replaceOperandWith(unsigned I, Metadata *New) { @@ -811,7 +836,7 @@ void MDNode::setOperand(unsigned I, Metadata *New) { mutable_begin()[I].reset(New, isUniqued() ? this : nullptr); } -/// \brief Get a node, or a self-reference that looks like it. +/// Get a node or a self-reference that looks like it. /// /// Special handling for finding self-references, for use by \a /// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from @@ -1100,6 +1125,43 @@ void MDAttachmentMap::getAll( array_pod_sort(Result.begin(), Result.end()); } +void MDGlobalAttachmentMap::insert(unsigned ID, MDNode &MD) { + Attachments.push_back({ID, TrackingMDNodeRef(&MD)}); +} + +void MDGlobalAttachmentMap::get(unsigned ID, + SmallVectorImpl<MDNode *> &Result) { + for (auto A : Attachments) + if (A.MDKind == ID) + Result.push_back(A.Node); +} + +void MDGlobalAttachmentMap::erase(unsigned ID) { + auto Follower = Attachments.begin(); + for (auto Leader = Attachments.begin(), E = Attachments.end(); Leader != E; + ++Leader) { + if (Leader->MDKind != ID) { + if (Follower != Leader) + *Follower = std::move(*Leader); + ++Follower; + } + } + Attachments.resize(Follower - Attachments.begin()); +} + +void MDGlobalAttachmentMap::getAll( + SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const { + for (auto &A : Attachments) + Result.emplace_back(A.MDKind, A.Node); + + // Sort the resulting array so it is stable with respect to metadata IDs. We + // need to preserve the original insertion order though. + std::stable_sort( + Result.begin(), Result.end(), + [](const std::pair<unsigned, MDNode *> &A, + const std::pair<unsigned, MDNode *> &B) { return A.first < B.first; }); +} + void Instruction::setMetadata(StringRef Kind, MDNode *Node) { if (!Node && !hasMetadata()) return; @@ -1138,9 +1200,6 @@ void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) { } } -/// setMetadata - Set the metadata of the specified kind to the specified -/// node. This updates/replaces metadata if already present, or removes it if -/// Node is null. void Instruction::setMetadata(unsigned KindID, MDNode *Node) { if (!Node && !hasMetadata()) return; @@ -1229,88 +1288,162 @@ void Instruction::getAllMetadataOtherThanDebugLocImpl( Info.getAll(Result); } -/// clearMetadataHashEntries - Clear all hashtable-based metadata from -/// this instruction. +bool Instruction::extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) { + assert((getOpcode() == Instruction::Br || + getOpcode() == Instruction::Select) && + "Looking for branch weights on something besides branch or select"); + + auto *ProfileData = getMetadata(LLVMContext::MD_prof); + if (!ProfileData || ProfileData->getNumOperands() != 3) + return false; + + auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); + if (!ProfDataName || !ProfDataName->getString().equals("branch_weights")) + return false; + + auto *CITrue = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(1)); + auto *CIFalse = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2)); + if (!CITrue || !CIFalse) + return false; + + TrueVal = CITrue->getValue().getZExtValue(); + FalseVal = CIFalse->getValue().getZExtValue(); + + return true; +} + +bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) { + assert((getOpcode() == Instruction::Br || + getOpcode() == Instruction::Select || + getOpcode() == Instruction::Call || + getOpcode() == Instruction::Invoke) && + "Looking for branch weights on something besides branch"); + + TotalVal = 0; + auto *ProfileData = getMetadata(LLVMContext::MD_prof); + if (!ProfileData) + return false; + + auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); + if (!ProfDataName || !ProfDataName->getString().equals("branch_weights")) + return false; + + TotalVal = 0; + for (unsigned i = 1; i < ProfileData->getNumOperands(); i++) { + auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(i)); + if (!V) + return false; + TotalVal += V->getValue().getZExtValue(); + } + return true; +} + void Instruction::clearMetadataHashEntries() { assert(hasMetadataHashEntry() && "Caller should check"); getContext().pImpl->InstructionMetadata.erase(this); setHasMetadataHashEntry(false); } -MDNode *Function::getMetadata(unsigned KindID) const { - if (!hasMetadata()) - return nullptr; - return getContext().pImpl->FunctionMetadata[this].lookup(KindID); +void GlobalObject::getMetadata(unsigned KindID, + SmallVectorImpl<MDNode *> &MDs) const { + if (hasMetadata()) + getContext().pImpl->GlobalObjectMetadata[this].get(KindID, MDs); } -MDNode *Function::getMetadata(StringRef Kind) const { - if (!hasMetadata()) - return nullptr; - return getMetadata(getContext().getMDKindID(Kind)); +void GlobalObject::getMetadata(StringRef Kind, + SmallVectorImpl<MDNode *> &MDs) const { + if (hasMetadata()) + getMetadata(getContext().getMDKindID(Kind), MDs); } -void Function::setMetadata(unsigned KindID, MDNode *MD) { - if (MD) { - if (!hasMetadata()) - setHasMetadataHashEntry(true); +void GlobalObject::addMetadata(unsigned KindID, MDNode &MD) { + if (!hasMetadata()) + setHasMetadataHashEntry(true); - getContext().pImpl->FunctionMetadata[this].set(KindID, *MD); - return; - } + getContext().pImpl->GlobalObjectMetadata[this].insert(KindID, MD); +} + +void GlobalObject::addMetadata(StringRef Kind, MDNode &MD) { + addMetadata(getContext().getMDKindID(Kind), MD); +} +void GlobalObject::eraseMetadata(unsigned KindID) { // Nothing to unset. if (!hasMetadata()) return; - auto &Store = getContext().pImpl->FunctionMetadata[this]; + auto &Store = getContext().pImpl->GlobalObjectMetadata[this]; Store.erase(KindID); if (Store.empty()) clearMetadata(); } -void Function::setMetadata(StringRef Kind, MDNode *MD) { - if (!MD && !hasMetadata()) - return; - setMetadata(getContext().getMDKindID(Kind), MD); -} - -void Function::getAllMetadata( +void GlobalObject::getAllMetadata( SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const { MDs.clear(); if (!hasMetadata()) return; - getContext().pImpl->FunctionMetadata[this].getAll(MDs); + getContext().pImpl->GlobalObjectMetadata[this].getAll(MDs); } -void Function::dropUnknownMetadata(ArrayRef<unsigned> KnownIDs) { +void GlobalObject::clearMetadata() { if (!hasMetadata()) return; - if (KnownIDs.empty()) { - clearMetadata(); - return; - } + getContext().pImpl->GlobalObjectMetadata.erase(this); + setHasMetadataHashEntry(false); +} - SmallSet<unsigned, 5> KnownSet; - KnownSet.insert(KnownIDs.begin(), KnownIDs.end()); +void GlobalObject::setMetadata(unsigned KindID, MDNode *N) { + eraseMetadata(KindID); + if (N) + addMetadata(KindID, *N); +} - auto &Store = getContext().pImpl->FunctionMetadata[this]; - assert(!Store.empty()); +void GlobalObject::setMetadata(StringRef Kind, MDNode *N) { + setMetadata(getContext().getMDKindID(Kind), N); +} - Store.remove_if([&KnownSet](const std::pair<unsigned, TrackingMDNodeRef> &I) { - return !KnownSet.count(I.first); - }); +MDNode *GlobalObject::getMetadata(unsigned KindID) const { + SmallVector<MDNode *, 1> MDs; + getMetadata(KindID, MDs); + assert(MDs.size() <= 1 && "Expected at most one metadata attachment"); + if (MDs.empty()) + return nullptr; + return MDs[0]; +} - if (Store.empty()) - clearMetadata(); +MDNode *GlobalObject::getMetadata(StringRef Kind) const { + return getMetadata(getContext().getMDKindID(Kind)); } -void Function::clearMetadata() { - if (!hasMetadata()) - return; - getContext().pImpl->FunctionMetadata.erase(this); - setHasMetadataHashEntry(false); +void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) { + SmallVector<std::pair<unsigned, MDNode *>, 8> MDs; + Other->getAllMetadata(MDs); + for (auto &MD : MDs) { + // We need to adjust the type metadata offset. + if (Offset != 0 && MD.first == LLVMContext::MD_type) { + auto *OffsetConst = cast<ConstantInt>( + cast<ConstantAsMetadata>(MD.second->getOperand(0))->getValue()); + Metadata *TypeId = MD.second->getOperand(1); + auto *NewOffsetMD = ConstantAsMetadata::get(ConstantInt::get( + OffsetConst->getType(), OffsetConst->getValue() + Offset)); + addMetadata(LLVMContext::MD_type, + *MDNode::get(getContext(), {NewOffsetMD, TypeId})); + continue; + } + addMetadata(MD.first, *MD.second); + } +} + +void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) { + addMetadata( + LLVMContext::MD_type, + *MDTuple::get(getContext(), + {llvm::ConstantAsMetadata::get(llvm::ConstantInt::get( + Type::getInt64Ty(getContext()), Offset)), + TypeID})); } void Function::setSubprogram(DISubprogram *SP) { diff --git a/lib/IR/Module.cpp b/lib/IR/Module.cpp index ac578d6dba0fa..ae81b25b95178 100644 --- a/lib/IR/Module.cpp +++ b/lib/IR/Module.cpp @@ -13,12 +13,13 @@ #include "llvm/IR/Module.h" #include "SymbolTableListTraitsImpl.h" -#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/GVMaterializer.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/LLVMContext.h" @@ -41,13 +42,14 @@ using namespace llvm; template class llvm::SymbolTableListTraits<Function>; template class llvm::SymbolTableListTraits<GlobalVariable>; template class llvm::SymbolTableListTraits<GlobalAlias>; +template class llvm::SymbolTableListTraits<GlobalIFunc>; //===----------------------------------------------------------------------===// // Primitive Module methods. // Module::Module(StringRef MID, LLVMContext &C) - : Context(C), Materializer(), ModuleID(MID), DL("") { + : Context(C), Materializer(), ModuleID(MID), SourceFileName(MID), DL("") { ValSymTab = new ValueSymbolTable(); NamedMDSymTab = new StringMap<NamedMDNode *>(); Context.addModule(this); @@ -59,6 +61,7 @@ Module::~Module() { GlobalList.clear(); FunctionList.clear(); AliasList.clear(); + IFuncList.clear(); NamedMDList.clear(); delete ValSymTab; delete static_cast<StringMap<NamedMDNode *> *>(NamedMDSymTab); @@ -250,6 +253,10 @@ GlobalAlias *Module::getNamedAlias(StringRef Name) const { return dyn_cast_or_null<GlobalAlias>(getNamedValue(Name)); } +GlobalIFunc *Module::getNamedIFunc(StringRef Name) const { + return dyn_cast_or_null<GlobalIFunc>(getNamedValue(Name)); +} + /// getNamedMetadata - Return the first NamedMDNode in the module with the /// specified name. This method returns null if a NamedMDNode with the /// specified name is not found. @@ -374,6 +381,19 @@ void Module::setDataLayout(const DataLayout &Other) { DL = Other; } const DataLayout &Module::getDataLayout() const { return DL; } +DICompileUnit *Module::debug_compile_units_iterator::operator*() const { + return cast<DICompileUnit>(CUs->getOperand(Idx)); +} +DICompileUnit *Module::debug_compile_units_iterator::operator->() const { + return cast<DICompileUnit>(CUs->getOperand(Idx)); +} + +void Module::debug_compile_units_iterator::SkipNoDebugCUs() { + while (CUs && (Idx < CUs->getNumOperands()) && + ((*this)->getEmissionKind() == DICompileUnit::NoDebug)) + ++Idx; +} + //===----------------------------------------------------------------------===// // Methods to control the materialization of GlobalValues in the Module. // @@ -438,6 +458,9 @@ void Module::dropAllReferences() { for (GlobalAlias &GA : aliases()) GA.dropAllReferences(); + + for (GlobalIFunc &GIF : ifuncs()) + GIF.dropAllReferences(); } unsigned Module::getDwarfVersion() const { @@ -464,7 +487,7 @@ PICLevel::Level Module::getPICLevel() const { auto *Val = cast_or_null<ConstantAsMetadata>(getModuleFlag("PIC Level")); if (!Val) - return PICLevel::Default; + return PICLevel::NotPIC; return static_cast<PICLevel::Level>( cast<ConstantInt>(Val->getValue())->getZExtValue()); @@ -474,14 +497,39 @@ void Module::setPICLevel(PICLevel::Level PL) { addModuleFlag(ModFlagBehavior::Error, "PIC Level", PL); } -void Module::setMaximumFunctionCount(uint64_t Count) { - addModuleFlag(ModFlagBehavior::Error, "MaxFunctionCount", Count); -} +PIELevel::Level Module::getPIELevel() const { + auto *Val = cast_or_null<ConstantAsMetadata>(getModuleFlag("PIE Level")); -Optional<uint64_t> Module::getMaximumFunctionCount() { - auto *Val = - cast_or_null<ConstantAsMetadata>(getModuleFlag("MaxFunctionCount")); if (!Val) - return None; - return cast<ConstantInt>(Val->getValue())->getZExtValue(); + return PIELevel::Default; + + return static_cast<PIELevel::Level>( + cast<ConstantInt>(Val->getValue())->getZExtValue()); +} + +void Module::setPIELevel(PIELevel::Level PL) { + addModuleFlag(ModFlagBehavior::Error, "PIE Level", PL); +} + +void Module::setProfileSummary(Metadata *M) { + addModuleFlag(ModFlagBehavior::Error, "ProfileSummary", M); +} + +Metadata *Module::getProfileSummary() { + return getModuleFlag("ProfileSummary"); +} + +GlobalVariable *llvm::collectUsedGlobalVariables( + const Module &M, SmallPtrSetImpl<GlobalValue *> &Set, bool CompilerUsed) { + const char *Name = CompilerUsed ? "llvm.compiler.used" : "llvm.used"; + GlobalVariable *GV = M.getGlobalVariable(Name); + if (!GV || !GV->hasInitializer()) + return GV; + + const ConstantArray *Init = cast<ConstantArray>(GV->getInitializer()); + for (Value *Op : Init->operands()) { + GlobalValue *G = cast<GlobalValue>(Op->stripPointerCastsNoFollowAliases()); + Set.insert(G); + } + return GV; } diff --git a/lib/IR/ModuleSummaryIndex.cpp b/lib/IR/ModuleSummaryIndex.cpp new file mode 100644 index 0000000000000..6107cf40a08da --- /dev/null +++ b/lib/IR/ModuleSummaryIndex.cpp @@ -0,0 +1,107 @@ +//===-- ModuleSummaryIndex.cpp - Module Summary Index ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the module index and summary classes for the +// IR library. +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/ADT/StringMap.h" +using namespace llvm; + +// Create the combined module index/summary from multiple +// per-module instances. +void ModuleSummaryIndex::mergeFrom(std::unique_ptr<ModuleSummaryIndex> Other, + uint64_t NextModuleId) { + + StringRef ModPath; + for (auto &OtherGlobalValSummaryLists : *Other) { + GlobalValue::GUID ValueGUID = OtherGlobalValSummaryLists.first; + GlobalValueSummaryList &List = OtherGlobalValSummaryLists.second; + + // Assert that the value summary list only has one entry, since we shouldn't + // have duplicate names within a single per-module index. + assert(List.size() == 1); + std::unique_ptr<GlobalValueSummary> Summary = std::move(List.front()); + + // Add the module path string ref for this module if we haven't already + // saved a reference to it. + if (ModPath.empty()) { + auto Path = Summary->modulePath(); + ModPath = addModulePath(Path, NextModuleId, Other->getModuleHash(Path)) + ->first(); + } else + assert(ModPath == Summary->modulePath() && + "Each module in the combined map should have a unique ID"); + + // Note the module path string ref was copied above and is still owned by + // the original per-module index. Reset it to the new module path + // string reference owned by the combined index. + Summary->setModulePath(ModPath); + + // Add new value summary to existing list. There may be duplicates when + // combining GlobalValueMap entries, due to COMDAT values. Any local + // values were given unique global IDs. + addGlobalValueSummary(ValueGUID, std::move(Summary)); + } +} + +void ModuleSummaryIndex::removeEmptySummaryEntries() { + for (auto MI = begin(), MIE = end(); MI != MIE;) { + // Only expect this to be called on a per-module index, which has a single + // entry per value entry list. + assert(MI->second.size() == 1); + if (!MI->second[0]) + MI = GlobalValueMap.erase(MI); + else + ++MI; + } +} + +// Collect for the given module the list of function it defines +// (GUID -> Summary). +void ModuleSummaryIndex::collectDefinedFunctionsForModule( + StringRef ModulePath, GVSummaryMapTy &GVSummaryMap) const { + for (auto &GlobalList : *this) { + auto GUID = GlobalList.first; + for (auto &GlobSummary : GlobalList.second) { + auto *Summary = dyn_cast_or_null<FunctionSummary>(GlobSummary.get()); + if (!Summary) + // Ignore global variable, focus on functions + continue; + // Ignore summaries from other modules. + if (Summary->modulePath() != ModulePath) + continue; + GVSummaryMap[GUID] = Summary; + } + } +} + +// Collect for each module the list of function it defines (GUID -> Summary). +void ModuleSummaryIndex::collectDefinedGVSummariesPerModule( + StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries) const { + for (auto &GlobalList : *this) { + auto GUID = GlobalList.first; + for (auto &Summary : GlobalList.second) { + ModuleToDefinedGVSummaries[Summary->modulePath()][GUID] = Summary.get(); + } + } +} + +GlobalValueSummary * +ModuleSummaryIndex::getGlobalValueSummary(uint64_t ValueGUID, + bool PerModuleIndex) const { + auto SummaryList = findGlobalValueSummaryList(ValueGUID); + assert(SummaryList != end() && "GlobalValue not found in index"); + assert((!PerModuleIndex || SummaryList->second.size() == 1) && + "Expected a single entry per global value in per-module index"); + auto &Summary = SummaryList->second[0]; + return Summary.get(); +} diff --git a/lib/IR/Operator.cpp b/lib/IR/Operator.cpp index 77dc680af1108..8a94053a72c50 100644 --- a/lib/IR/Operator.cpp +++ b/lib/IR/Operator.cpp @@ -12,6 +12,12 @@ Type *GEPOperator::getSourceElementType() const { return cast<GetElementPtrConstantExpr>(this)->getSourceElementType(); } +Type *GEPOperator::getResultElementType() const { + if (auto *I = dyn_cast<GetElementPtrInst>(this)) + return I->getResultElementType(); + return cast<GetElementPtrConstantExpr>(this)->getResultElementType(); +} + bool GEPOperator::accumulateConstantOffset(const DataLayout &DL, APInt &Offset) const { assert(Offset.getBitWidth() == diff --git a/lib/IR/OptBisect.cpp b/lib/IR/OptBisect.cpp new file mode 100644 index 0000000000000..e9574ca81261c --- /dev/null +++ b/lib/IR/OptBisect.cpp @@ -0,0 +1,120 @@ +//===------- llvm/IR/OptBisect/Bisect.cpp - LLVM Bisect support --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements support for a bisecting optimizations based on a +/// command line option. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/OptBisect.h" +#include "llvm/Pass.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; + +static cl::opt<int> OptBisectLimit("opt-bisect-limit", cl::Hidden, + cl::init(INT_MAX), cl::Optional, + cl::desc("Maximum optimization to perform")); + +OptBisect::OptBisect() { + BisectEnabled = OptBisectLimit != INT_MAX; +} + +static void printPassMessage(const StringRef &Name, int PassNum, + StringRef TargetDesc, bool Running) { + StringRef Status = Running ? "" : "NOT "; + errs() << "BISECT: " << Status << "running pass " + << "(" << PassNum << ") " << Name << " on " << TargetDesc << "\n"; +} + +static void printCaseMessage(int CaseNum, StringRef Msg, bool Running) { + if (Running) + errs() << "BISECT: running case ("; + else + errs() << "BISECT: NOT running case ("; + errs() << CaseNum << "): " << Msg << "\n"; +} + +static std::string getDescription(const Module &M) { + return "module (" + M.getName().str() + ")"; +} + +static std::string getDescription(const Function &F) { + return "function (" + F.getName().str() + ")"; +} + +static std::string getDescription(const BasicBlock &BB) { + return "basic block (" + BB.getName().str() + ") in function (" + + BB.getParent()->getName().str() + ")"; +} + +static std::string getDescription(const Loop &L) { + // FIXME: I'd like to be able to provide a better description here, but + // calling L->getHeader() would introduce a new dependency on the + // LLVMCore library. + return "loop"; +} + +static std::string getDescription(const CallGraphSCC &SCC) { + std::string Desc = "SCC ("; + bool First = true; + for (CallGraphNode *CGN : SCC) { + if (First) + First = false; + else + Desc += ", "; + Function *F = CGN->getFunction(); + if (F) + Desc += F->getName(); + else + Desc += "<<null function>>"; + } + Desc += ")"; + return Desc; +} + +// Force instantiations. +template bool OptBisect::shouldRunPass(const Pass *, const Module &); +template bool OptBisect::shouldRunPass(const Pass *, const Function &); +template bool OptBisect::shouldRunPass(const Pass *, const BasicBlock &); +template bool OptBisect::shouldRunPass(const Pass *, const Loop &); +template bool OptBisect::shouldRunPass(const Pass *, const CallGraphSCC &); + +template <class UnitT> +bool OptBisect::shouldRunPass(const Pass *P, const UnitT &U) { + if (!BisectEnabled) + return true; + return checkPass(P->getPassName(), getDescription(U)); +} + +bool OptBisect::checkPass(const StringRef PassName, + const StringRef TargetDesc) { + assert(BisectEnabled); + + int CurBisectNum = ++LastBisectNum; + bool ShouldRun = (OptBisectLimit == -1 || CurBisectNum <= OptBisectLimit); + printPassMessage(PassName, CurBisectNum, TargetDesc, ShouldRun); + return ShouldRun; +} + +bool OptBisect::shouldRunCase(const Twine &Msg) { + if (!BisectEnabled) + return true; + int CurFuelNum = ++LastBisectNum; + bool ShouldRun = (OptBisectLimit == -1 || CurFuelNum <= OptBisectLimit); + printCaseMessage(CurFuelNum, Msg.str(), ShouldRun); + return ShouldRun; +} + diff --git a/lib/IR/Pass.cpp b/lib/IR/Pass.cpp index df45460a6ccac..69299fea7634c 100644 --- a/lib/IR/Pass.cpp +++ b/lib/IR/Pass.cpp @@ -17,6 +17,8 @@ #include "llvm/IR/Function.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/LegacyPassNameParser.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/OptBisect.h" #include "llvm/PassRegistry.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -45,6 +47,10 @@ PassManagerType ModulePass::getPotentialPassManagerType() const { return PMT_ModulePassManager; } +bool ModulePass::skipModule(Module &M) const { + return !M.getContext().getOptBisect().shouldRunPass(this, M); +} + bool Pass::mustPreserveAnalysisID(char &AID) const { return Resolver->getAnalysisIfAvailable(&AID, true) != nullptr; } @@ -113,7 +119,7 @@ void Pass::print(raw_ostream &O,const Module*) const { } // dump - call print(cerr); -void Pass::dump() const { +LLVM_DUMP_METHOD void Pass::dump() const { print(dbgs(), nullptr); } @@ -140,10 +146,13 @@ PassManagerType FunctionPass::getPotentialPassManagerType() const { return PMT_FunctionPassManager; } -bool FunctionPass::skipOptnoneFunction(const Function &F) const { +bool FunctionPass::skipFunction(const Function &F) const { + if (!F.getContext().getOptBisect().shouldRunPass(this, F)) + return true; + if (F.hasFnAttribute(Attribute::OptimizeNone)) { - DEBUG(dbgs() << "Skipping pass '" << getPassName() - << "' on function " << F.getName() << "\n"); + DEBUG(dbgs() << "Skipping pass '" << getPassName() << "' on function " + << F.getName() << "\n"); return true; } return false; @@ -168,9 +177,13 @@ bool BasicBlockPass::doFinalization(Function &) { return false; } -bool BasicBlockPass::skipOptnoneFunction(const BasicBlock &BB) const { +bool BasicBlockPass::skipBasicBlock(const BasicBlock &BB) const { const Function *F = BB.getParent(); - if (F && F->hasFnAttribute(Attribute::OptimizeNone)) { + if (!F) + return false; + if (!F->getContext().getOptBisect().shouldRunPass(this, BB)) + return true; + if (F->hasFnAttribute(Attribute::OptimizeNone)) { // Report this only once per function. if (&BB == &F->getEntryBlock()) DEBUG(dbgs() << "Skipping pass '" << getPassName() diff --git a/lib/IR/PassManager.cpp b/lib/IR/PassManager.cpp index a5f407c00e8e1..8563a4019a2b8 100644 --- a/lib/IR/PassManager.cpp +++ b/lib/IR/PassManager.cpp @@ -7,37 +7,18 @@ // //===----------------------------------------------------------------------===// +#include "llvm/IR/PassManager.h" #include "llvm/ADT/STLExtras.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/PassManager.h" using namespace llvm; -char FunctionAnalysisManagerModuleProxy::PassID; - -FunctionAnalysisManagerModuleProxy::Result -FunctionAnalysisManagerModuleProxy::run(Module &M) { - assert(FAM->empty() && "Function analyses ran prior to the module proxy!"); - return Result(*FAM); -} - -FunctionAnalysisManagerModuleProxy::Result::~Result() { - // Clear out the analysis manager if we're being destroyed -- it means we - // didn't even see an invalidate call when we got invalidated. - FAM->clear(); +// Explicit template instantiations for core template typedefs. +namespace llvm { +template class PassManager<Module>; +template class PassManager<Function>; +template class AnalysisManager<Module>; +template class AnalysisManager<Function>; +template class InnerAnalysisManagerProxy<FunctionAnalysisManager, Module>; +template class OuterAnalysisManagerProxy<ModuleAnalysisManager, Function>; } - -bool FunctionAnalysisManagerModuleProxy::Result::invalidate( - Module &M, const PreservedAnalyses &PA) { - // If this proxy isn't marked as preserved, then we can't even invalidate - // individual function analyses, there may be an invalid set of Function - // objects in the cache making it impossible to incrementally preserve them. - // Just clear the entire manager. - if (!PA.preserved(ID())) - FAM->clear(); - - // Return false to indicate that this result is still a valid proxy. - return false; -} - -char ModuleAnalysisManagerFunctionProxy::PassID; diff --git a/lib/IR/PassRegistry.cpp b/lib/IR/PassRegistry.cpp index b879fef3f4a72..09b17ba308dd0 100644 --- a/lib/IR/PassRegistry.cpp +++ b/lib/IR/PassRegistry.cpp @@ -13,12 +13,8 @@ //===----------------------------------------------------------------------===// #include "llvm/PassRegistry.h" -#include "llvm/IR/Function.h" #include "llvm/PassSupport.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/ManagedStatic.h" -#include "llvm/Support/RWMutex.h" -#include <vector> using namespace llvm; diff --git a/lib/IR/ProfileSummary.cpp b/lib/IR/ProfileSummary.cpp new file mode 100644 index 0000000000000..2b24d1251121a --- /dev/null +++ b/lib/IR/ProfileSummary.cpp @@ -0,0 +1,191 @@ +//=-- Profilesummary.cpp - Profile summary support --------------------------=// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains support for converting profile summary data from/to +// metadata. +// +//===----------------------------------------------------------------------===// + +#include "llvm/IR/ProfileSummary.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Type.h" +#include "llvm/Support/Casting.h" + +using namespace llvm; + +const char *ProfileSummary::KindStr[2] = {"InstrProf", "SampleProfile"}; + +// Return an MDTuple with two elements. The first element is a string Key and +// the second is a uint64_t Value. +static Metadata *getKeyValMD(LLVMContext &Context, const char *Key, + uint64_t Val) { + Type *Int64Ty = Type::getInt64Ty(Context); + Metadata *Ops[2] = {MDString::get(Context, Key), + ConstantAsMetadata::get(ConstantInt::get(Int64Ty, Val))}; + return MDTuple::get(Context, Ops); +} + +// Return an MDTuple with two elements. The first element is a string Key and +// the second is a string Value. +static Metadata *getKeyValMD(LLVMContext &Context, const char *Key, + const char *Val) { + Metadata *Ops[2] = {MDString::get(Context, Key), MDString::get(Context, Val)}; + return MDTuple::get(Context, Ops); +} + +// This returns an MDTuple representing the detiled summary. The tuple has two +// elements: a string "DetailedSummary" and an MDTuple representing the value +// of the detailed summary. Each element of this tuple is again an MDTuple whose +// elements are the (Cutoff, MinCount, NumCounts) triplet of the +// DetailedSummaryEntry. +Metadata *ProfileSummary::getDetailedSummaryMD(LLVMContext &Context) { + std::vector<Metadata *> Entries; + Type *Int32Ty = Type::getInt32Ty(Context); + Type *Int64Ty = Type::getInt64Ty(Context); + for (auto &Entry : DetailedSummary) { + Metadata *EntryMD[3] = { + ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Entry.Cutoff)), + ConstantAsMetadata::get(ConstantInt::get(Int64Ty, Entry.MinCount)), + ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Entry.NumCounts))}; + Entries.push_back(MDTuple::get(Context, EntryMD)); + } + Metadata *Ops[2] = {MDString::get(Context, "DetailedSummary"), + MDTuple::get(Context, Entries)}; + return MDTuple::get(Context, Ops); +} + +// This returns an MDTuple representing this ProfileSummary object. The first +// entry of this tuple is another MDTuple of two elements: a string +// "ProfileFormat" and a string representing the format ("InstrProf" or +// "SampleProfile"). The rest of the elements of the outer MDTuple are specific +// to the kind of profile summary as returned by getFormatSpecificMD. +Metadata *ProfileSummary::getMD(LLVMContext &Context) { + std::vector<Metadata *> Components; + Components.push_back(getKeyValMD(Context, "ProfileFormat", KindStr[PSK])); + + Components.push_back(getKeyValMD(Context, "TotalCount", getTotalCount())); + Components.push_back(getKeyValMD(Context, "MaxCount", getMaxCount())); + Components.push_back( + getKeyValMD(Context, "MaxInternalCount", getMaxInternalCount())); + Components.push_back( + getKeyValMD(Context, "MaxFunctionCount", getMaxFunctionCount())); + Components.push_back(getKeyValMD(Context, "NumCounts", getNumCounts())); + Components.push_back(getKeyValMD(Context, "NumFunctions", getNumFunctions())); + Components.push_back(getDetailedSummaryMD(Context)); + return MDTuple::get(Context, Components); +} + +// Parse an MDTuple representing (Key, Val) pair. +static bool getVal(MDTuple *MD, const char *Key, uint64_t &Val) { + if (!MD) + return false; + if (MD->getNumOperands() != 2) + return false; + MDString *KeyMD = dyn_cast<MDString>(MD->getOperand(0)); + ConstantAsMetadata *ValMD = dyn_cast<ConstantAsMetadata>(MD->getOperand(1)); + if (!KeyMD || !ValMD) + return false; + if (!KeyMD->getString().equals(Key)) + return false; + Val = cast<ConstantInt>(ValMD->getValue())->getZExtValue(); + return true; +} + +// Check if an MDTuple represents a (Key, Val) pair. +static bool isKeyValuePair(MDTuple *MD, const char *Key, const char *Val) { + if (!MD || MD->getNumOperands() != 2) + return false; + MDString *KeyMD = dyn_cast<MDString>(MD->getOperand(0)); + MDString *ValMD = dyn_cast<MDString>(MD->getOperand(1)); + if (!KeyMD || !ValMD) + return false; + if (!KeyMD->getString().equals(Key) || !ValMD->getString().equals(Val)) + return false; + return true; +} + +// Parse an MDTuple representing detailed summary. +static bool getSummaryFromMD(MDTuple *MD, SummaryEntryVector &Summary) { + if (!MD || MD->getNumOperands() != 2) + return false; + MDString *KeyMD = dyn_cast<MDString>(MD->getOperand(0)); + if (!KeyMD || !KeyMD->getString().equals("DetailedSummary")) + return false; + MDTuple *EntriesMD = dyn_cast<MDTuple>(MD->getOperand(1)); + if (!EntriesMD) + return false; + for (auto &&MDOp : EntriesMD->operands()) { + MDTuple *EntryMD = dyn_cast<MDTuple>(MDOp); + if (!EntryMD || EntryMD->getNumOperands() != 3) + return false; + ConstantAsMetadata *Op0 = + dyn_cast<ConstantAsMetadata>(EntryMD->getOperand(0)); + ConstantAsMetadata *Op1 = + dyn_cast<ConstantAsMetadata>(EntryMD->getOperand(1)); + ConstantAsMetadata *Op2 = + dyn_cast<ConstantAsMetadata>(EntryMD->getOperand(2)); + + if (!Op0 || !Op1 || !Op2) + return false; + Summary.emplace_back(cast<ConstantInt>(Op0->getValue())->getZExtValue(), + cast<ConstantInt>(Op1->getValue())->getZExtValue(), + cast<ConstantInt>(Op2->getValue())->getZExtValue()); + } + return true; +} + +ProfileSummary *ProfileSummary::getFromMD(Metadata *MD) { + if (!MD) + return nullptr; + if (!isa<MDTuple>(MD)) + return nullptr; + MDTuple *Tuple = cast<MDTuple>(MD); + if (Tuple->getNumOperands() != 8) + return nullptr; + + auto &FormatMD = Tuple->getOperand(0); + ProfileSummary::Kind SummaryKind; + if (isKeyValuePair(dyn_cast_or_null<MDTuple>(FormatMD), "ProfileFormat", + "SampleProfile")) + SummaryKind = PSK_Sample; + else if (isKeyValuePair(dyn_cast_or_null<MDTuple>(FormatMD), "ProfileFormat", + "InstrProf")) + SummaryKind = PSK_Instr; + else + return nullptr; + + uint64_t NumCounts, TotalCount, NumFunctions, MaxFunctionCount, MaxCount, + MaxInternalCount; + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(1)), "TotalCount", + TotalCount)) + return nullptr; + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(2)), "MaxCount", MaxCount)) + return nullptr; + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(3)), "MaxInternalCount", + MaxInternalCount)) + return nullptr; + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(4)), "MaxFunctionCount", + MaxFunctionCount)) + return nullptr; + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(5)), "NumCounts", NumCounts)) + return nullptr; + if (!getVal(dyn_cast<MDTuple>(Tuple->getOperand(6)), "NumFunctions", + NumFunctions)) + return nullptr; + + SummaryEntryVector Summary; + if (!getSummaryFromMD(dyn_cast<MDTuple>(Tuple->getOperand(7)), Summary)) + return nullptr; + return new ProfileSummary(SummaryKind, Summary, TotalCount, MaxCount, + MaxInternalCount, MaxFunctionCount, NumCounts, + NumFunctions); +} diff --git a/lib/IR/Statepoint.cpp b/lib/IR/Statepoint.cpp index 27a990eaff811..63be1e780d814 100644 --- a/lib/IR/Statepoint.cpp +++ b/lib/IR/Statepoint.cpp @@ -7,55 +7,68 @@ // //===----------------------------------------------------------------------===// // -// +// This file contains some utility functions to help recognize gc.statepoint +// intrinsics. +// //===----------------------------------------------------------------------===// -#include "llvm/IR/Function.h" -#include "llvm/IR/Constant.h" -#include "llvm/IR/Constants.h" #include "llvm/IR/Statepoint.h" -#include "llvm/Support/CommandLine.h" -using namespace std; +#include "llvm/IR/Function.h" + using namespace llvm; -bool llvm::isStatepoint(const ImmutableCallSite &CS) { - if (!CS.getInstruction()) { - // This is not a call site - return false; - } +static const Function *getCalledFunction(ImmutableCallSite CS) { + if (!CS.getInstruction()) + return nullptr; + return CS.getCalledFunction(); +} - const Function *F = CS.getCalledFunction(); - return (F && F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint); +bool llvm::isStatepoint(ImmutableCallSite CS) { + if (auto *F = getCalledFunction(CS)) + return F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint; + return false; } -bool llvm::isStatepoint(const Value *inst) { - if (isa<InvokeInst>(inst) || isa<CallInst>(inst)) { - ImmutableCallSite CS(inst); + +bool llvm::isStatepoint(const Value *V) { + if (auto CS = ImmutableCallSite(V)) return isStatepoint(CS); - } return false; } -bool llvm::isStatepoint(const Value &inst) { - return isStatepoint(&inst); + +bool llvm::isStatepoint(const Value &V) { + return isStatepoint(&V); } -bool llvm::isGCRelocate(const ImmutableCallSite &CS) { +bool llvm::isGCRelocate(ImmutableCallSite CS) { return CS.getInstruction() && isa<GCRelocateInst>(CS.getInstruction()); } -bool llvm::isGCResult(const ImmutableCallSite &CS) { - if (!CS.getInstruction()) { - // This is not a call site - return false; - } +bool llvm::isGCResult(ImmutableCallSite CS) { + return CS.getInstruction() && isa<GCResultInst>(CS.getInstruction()); +} - return isGCResult(CS.getInstruction()); +bool llvm::isStatepointDirectiveAttr(Attribute Attr) { + return Attr.hasAttribute("statepoint-id") || + Attr.hasAttribute("statepoint-num-patch-bytes"); } -bool llvm::isGCResult(const Value *inst) { - if (const CallInst *call = dyn_cast<CallInst>(inst)) { - if (Function *F = call->getCalledFunction()) { - return F->getIntrinsicID() == Intrinsic::experimental_gc_result; - } - } - return false; + +StatepointDirectives llvm::parseStatepointDirectivesFromAttrs(AttributeSet AS) { + StatepointDirectives Result; + + Attribute AttrID = + AS.getAttribute(AttributeSet::FunctionIndex, "statepoint-id"); + uint64_t StatepointID; + if (AttrID.isStringAttribute()) + if (!AttrID.getValueAsString().getAsInteger(10, StatepointID)) + Result.StatepointID = StatepointID; + + uint32_t NumPatchBytes; + Attribute AttrNumPatchBytes = AS.getAttribute(AttributeSet::FunctionIndex, + "statepoint-num-patch-bytes"); + if (AttrNumPatchBytes.isStringAttribute()) + if (!AttrNumPatchBytes.getValueAsString().getAsInteger(10, NumPatchBytes)) + Result.NumPatchBytes = NumPatchBytes; + + return Result; } diff --git a/lib/IR/Type.cpp b/lib/IR/Type.cpp index 4c1baf52a58fd..5c97a4ea07050 100644 --- a/lib/IR/Type.cpp +++ b/lib/IR/Type.cpp @@ -41,22 +41,16 @@ Type *Type::getPrimitiveType(LLVMContext &C, TypeID IDNumber) { } } -/// getScalarType - If this is a vector type, return the element type, -/// otherwise return this. Type *Type::getScalarType() const { if (auto *VTy = dyn_cast<VectorType>(this)) return VTy->getElementType(); return const_cast<Type*>(this); } -/// isIntegerTy - Return true if this is an IntegerType of the specified width. bool Type::isIntegerTy(unsigned Bitwidth) const { return isIntegerTy() && cast<IntegerType>(this)->getBitWidth() == Bitwidth; } -// canLosslesslyBitCastTo - Return true if this type can be converted to -// 'Ty' without any reinterpretation of bits. For example, i8* to i32*. -// bool Type::canLosslesslyBitCastTo(Type *Ty) const { // Identity cast means no change so return true if (this == Ty) @@ -126,16 +120,10 @@ unsigned Type::getPrimitiveSizeInBits() const { } } -/// getScalarSizeInBits - If this is a vector type, return the -/// getPrimitiveSizeInBits value for the element type. Otherwise return the -/// getPrimitiveSizeInBits value for this type. unsigned Type::getScalarSizeInBits() const { return getScalarType()->getPrimitiveSizeInBits(); } -/// getFPMantissaWidth - Return the width of the mantissa of this type. This -/// is only valid on floating point types. If the FP type does not -/// have a stable mantissa (e.g. ppc long double), this method returns -1. int Type::getFPMantissaWidth() const { if (auto *VTy = dyn_cast<VectorType>(this)) return VTy->getElementType()->getFPMantissaWidth(); @@ -149,9 +137,6 @@ int Type::getFPMantissaWidth() const { return -1; } -/// isSizedDerivedType - Derived types like structures and arrays are sized -/// iff all of the members of the type are sized as well. Since asking for -/// their size is relatively uncommon, move this operation out of line. bool Type::isSizedDerivedType(SmallPtrSetImpl<Type*> *Visited) const { if (auto *ATy = dyn_cast<ArrayType>(this)) return ATy->getElementType()->isSized(Visited); @@ -302,7 +287,7 @@ FunctionType::FunctionType(Type *Result, ArrayRef<Type*> Params, NumContainedTys = Params.size() + 1; // + 1 for result type } -// FunctionType::get - The factory function for the FunctionType class. +// This is the factory function for the FunctionType class. FunctionType *FunctionType::get(Type *ReturnType, ArrayRef<Type*> Params, bool isVarArg) { LLVMContextImpl *pImpl = ReturnType->getContext().pImpl; @@ -327,15 +312,11 @@ FunctionType *FunctionType::get(Type *Result, bool isVarArg) { return get(Result, None, isVarArg); } -/// isValidReturnType - Return true if the specified type is valid as a return -/// type. bool FunctionType::isValidReturnType(Type *RetTy) { return !RetTy->isFunctionTy() && !RetTy->isLabelTy() && !RetTy->isMetadataTy(); } -/// isValidArgumentType - Return true if the specified type is valid as an -/// argument type. bool FunctionType::isValidArgumentType(Type *ArgTy) { return ArgTy->isFirstClassType(); } @@ -552,8 +533,6 @@ bool StructType::isValidElementType(Type *ElemTy) { !ElemTy->isTokenTy(); } -/// isLayoutIdentical - Return true if this is layout identical to the -/// specified struct. bool StructType::isLayoutIdentical(StructType *Other) const { if (this == Other) return true; @@ -563,8 +542,6 @@ bool StructType::isLayoutIdentical(StructType *Other) const { return elements() == Other->elements(); } -/// getTypeByName - Return the type with the specified name, or null if there -/// is none by that name. StructType *Module::getTypeByName(StringRef Name) const { return getContext().pImpl->NamedStructTypes.lookup(Name); } diff --git a/lib/IR/TypeFinder.cpp b/lib/IR/TypeFinder.cpp index b5bdab0865b61..dc4c1cffb20c5 100644 --- a/lib/IR/TypeFinder.cpp +++ b/lib/IR/TypeFinder.cpp @@ -41,23 +41,19 @@ void TypeFinder::run(const Module &M, bool onlyNamed) { // Get types from functions. SmallVector<std::pair<unsigned, MDNode *>, 4> MDForInst; - for (Module::const_iterator FI = M.begin(), E = M.end(); FI != E; ++FI) { - incorporateType(FI->getType()); + for (const Function &FI : M) { + incorporateType(FI.getType()); - for (const Use &U : FI->operands()) + for (const Use &U : FI.operands()) incorporateValue(U.get()); // First incorporate the arguments. - for (Function::const_arg_iterator AI = FI->arg_begin(), - AE = FI->arg_end(); AI != AE; ++AI) + for (Function::const_arg_iterator AI = FI.arg_begin(), AE = FI.arg_end(); + AI != AE; ++AI) incorporateValue(&*AI); - for (Function::const_iterator BB = FI->begin(), E = FI->end(); - BB != E;++BB) - for (BasicBlock::const_iterator II = BB->begin(), - E = BB->end(); II != E; ++II) { - const Instruction &I = *II; - + for (const BasicBlock &BB : FI) + for (const Instruction &I : BB) { // Incorporate the type of the instruction. incorporateType(I.getType()); diff --git a/lib/IR/Value.cpp b/lib/IR/Value.cpp index 4d224a041349b..6ae37fa6f15a9 100644 --- a/lib/IR/Value.cpp +++ b/lib/IR/Value.cpp @@ -34,6 +34,7 @@ #include "llvm/Support/ManagedStatic.h" #include "llvm/Support/raw_ostream.h" #include <algorithm> + using namespace llvm; //===----------------------------------------------------------------------===// @@ -58,6 +59,8 @@ Value::Value(Type *ty, unsigned scid) (SubclassID < ConstantFirstVal || SubclassID > ConstantLastVal)) assert((VTy->isFirstClassType() || VTy->isVoidTy()) && "Cannot create non-first-class values except for constants!"); + static_assert(sizeof(Value) == 3 * sizeof(void *) + 2 * sizeof(unsigned), + "Value too big"); } Value::~Value() { @@ -195,6 +198,10 @@ StringRef Value::getName() const { } void Value::setNameImpl(const Twine &NewName) { + // Fast-path: LLVMContext can be set to strip out non-GlobalValue names + if (getContext().shouldDiscardValueNames() && !isa<GlobalValue>(this)) + return; + // Fast path for common IRBuilder case of setName("") when there is no name. if (NewName.isTriviallyEmpty() && !hasName()) return; @@ -358,7 +365,7 @@ static bool contains(Value *Expr, Value *V) { SmallPtrSet<ConstantExpr *, 4> Cache; return contains(Cache, CE, C); } -#endif +#endif // NDEBUG void Value::replaceAllUsesWith(Value *New) { assert(New && "Value::replaceAllUsesWith(<null>) is invalid!"); @@ -379,7 +386,7 @@ void Value::replaceAllUsesWith(Value *New) { // constant because they are uniqued. if (auto *C = dyn_cast<Constant>(U.getUser())) { if (!isa<GlobalValue>(C)) { - C->handleOperandChange(this, New, &U); + C->handleOperandChange(this, New); continue; } } @@ -410,7 +417,6 @@ void Value::replaceUsesOutsideBlock(Value *New, BasicBlock *BB) { continue; U.set(New); } - return; } namespace { @@ -454,10 +460,16 @@ static Value *stripPointerCastsAndOffsets(Value *V) { Operator::getOpcode(V) == Instruction::AddrSpaceCast) { V = cast<Operator>(V)->getOperand(0); } else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) { - if (StripKind == PSK_ZeroIndices || GA->mayBeOverridden()) + if (StripKind == PSK_ZeroIndices || GA->isInterposable()) return V; V = GA->getAliasee(); } else { + if (auto CS = CallSite(V)) + if (Value *RV = CS.getReturnedArgOperand()) { + V = RV; + continue; + } + return V; } assert(V->getType()->isPointerTy() && "Unexpected operand type!"); @@ -465,7 +477,7 @@ static Value *stripPointerCastsAndOffsets(Value *V) { return V; } -} // namespace +} // end anonymous namespace Value *Value::stripPointerCasts() { return stripPointerCastsAndOffsets<PSK_ZeroIndicesAndAliases>(this); @@ -507,6 +519,12 @@ Value *Value::stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, } else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) { V = GA->getAliasee(); } else { + if (auto CS = CallSite(V)) + if (Value *RV = CS.getReturnedArgOperand()) { + V = RV; + continue; + } + return V; } assert(V->getType()->isPointerTy() && "Unexpected operand type!"); @@ -519,6 +537,104 @@ Value *Value::stripInBoundsOffsets() { return stripPointerCastsAndOffsets<PSK_InBounds>(this); } +unsigned Value::getPointerDereferenceableBytes(const DataLayout &DL, + bool &CanBeNull) const { + assert(getType()->isPointerTy() && "must be pointer"); + + unsigned DerefBytes = 0; + CanBeNull = false; + if (const Argument *A = dyn_cast<Argument>(this)) { + DerefBytes = A->getDereferenceableBytes(); + if (DerefBytes == 0 && A->hasByValAttr() && A->getType()->isSized()) { + DerefBytes = DL.getTypeStoreSize(A->getType()); + CanBeNull = false; + } + if (DerefBytes == 0) { + DerefBytes = A->getDereferenceableOrNullBytes(); + CanBeNull = true; + } + } else if (auto CS = ImmutableCallSite(this)) { + DerefBytes = CS.getDereferenceableBytes(0); + if (DerefBytes == 0) { + DerefBytes = CS.getDereferenceableOrNullBytes(0); + CanBeNull = true; + } + } else if (const LoadInst *LI = dyn_cast<LoadInst>(this)) { + if (MDNode *MD = LI->getMetadata(LLVMContext::MD_dereferenceable)) { + ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0)); + DerefBytes = CI->getLimitedValue(); + } + if (DerefBytes == 0) { + if (MDNode *MD = + LI->getMetadata(LLVMContext::MD_dereferenceable_or_null)) { + ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0)); + DerefBytes = CI->getLimitedValue(); + } + CanBeNull = true; + } + } else if (auto *AI = dyn_cast<AllocaInst>(this)) { + if (AI->getAllocatedType()->isSized()) { + DerefBytes = DL.getTypeStoreSize(AI->getAllocatedType()); + CanBeNull = false; + } + } else if (auto *GV = dyn_cast<GlobalVariable>(this)) { + if (GV->getValueType()->isSized() && !GV->hasExternalWeakLinkage()) { + // TODO: Don't outright reject hasExternalWeakLinkage but set the + // CanBeNull flag. + DerefBytes = DL.getTypeStoreSize(GV->getValueType()); + CanBeNull = false; + } + } + return DerefBytes; +} + +unsigned Value::getPointerAlignment(const DataLayout &DL) const { + assert(getType()->isPointerTy() && "must be pointer"); + + unsigned Align = 0; + if (auto *GO = dyn_cast<GlobalObject>(this)) { + Align = GO->getAlignment(); + if (Align == 0) { + if (auto *GVar = dyn_cast<GlobalVariable>(GO)) { + Type *ObjectType = GVar->getValueType(); + if (ObjectType->isSized()) { + // If the object is defined in the current Module, we'll be giving + // it the preferred alignment. Otherwise, we have to assume that it + // may only have the minimum ABI alignment. + if (GVar->isStrongDefinitionForLinker()) + Align = DL.getPreferredAlignment(GVar); + else + Align = DL.getABITypeAlignment(ObjectType); + } + } + } + } else if (const Argument *A = dyn_cast<Argument>(this)) { + Align = A->getParamAlignment(); + + if (!Align && A->hasStructRetAttr()) { + // An sret parameter has at least the ABI alignment of the return type. + Type *EltTy = cast<PointerType>(A->getType())->getElementType(); + if (EltTy->isSized()) + Align = DL.getABITypeAlignment(EltTy); + } + } else if (const AllocaInst *AI = dyn_cast<AllocaInst>(this)) { + Align = AI->getAlignment(); + if (Align == 0) { + Type *AllocatedType = AI->getAllocatedType(); + if (AllocatedType->isSized()) + Align = DL.getPrefTypeAlignment(AllocatedType); + } + } else if (auto CS = ImmutableCallSite(this)) + Align = CS.getAttributes().getParamAlignment(AttributeSet::ReturnIndex); + else if (const LoadInst *LI = dyn_cast<LoadInst>(this)) + if (MDNode *MD = LI->getMetadata(LLVMContext::MD_align)) { + ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0)); + Align = CI->getLimitedValue(); + } + + return Align; +} + Value *Value::DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) { PHINode *PN = dyn_cast<PHINode>(this); @@ -644,7 +760,6 @@ void ValueHandleBase::RemoveFromUseList() { } } - void ValueHandleBase::ValueIsDeleted(Value *V) { assert(V->HasValueHandle && "Should only be called if ValueHandles present"); @@ -701,7 +816,6 @@ void ValueHandleBase::ValueIsDeleted(Value *V) { } } - void ValueHandleBase::ValueIsRAUWd(Value *Old, Value *New) { assert(Old->HasValueHandle &&"Should only be called if ValueHandles present"); assert(Old != New && "Changing value into itself!"); diff --git a/lib/IR/ValueSymbolTable.cpp b/lib/IR/ValueSymbolTable.cpp index deb6e7573e722..f6f1dd984e9f9 100644 --- a/lib/IR/ValueSymbolTable.cpp +++ b/lib/IR/ValueSymbolTable.cpp @@ -24,10 +24,10 @@ using namespace llvm; // Class destructor ValueSymbolTable::~ValueSymbolTable() { #ifndef NDEBUG // Only do this in -g mode... - for (iterator VI = vmap.begin(), VE = vmap.end(); VI != VE; ++VI) + for (const auto &VI : vmap) dbgs() << "Value still in symbol table! Type = '" - << *VI->getValue()->getType() << "' Name = '" - << VI->getKeyData() << "'\n"; + << *VI.getValue()->getType() << "' Name = '" << VI.getKeyData() + << "'\n"; assert(vmap.empty() && "Values remain in symbol table!"); #endif } @@ -97,11 +97,11 @@ ValueName *ValueSymbolTable::createValueName(StringRef Name, Value *V) { // dump - print out the symbol table // -void ValueSymbolTable::dump() const { +LLVM_DUMP_METHOD void ValueSymbolTable::dump() const { //DEBUG(dbgs() << "ValueSymbolTable:\n"); - for (const_iterator I = begin(), E = end(); I != E; ++I) { + for (const auto &I : *this) { //DEBUG(dbgs() << " '" << I->getKeyData() << "' = "); - I->getValue()->dump(); + I.getValue()->dump(); //DEBUG(dbgs() << "\n"); } } diff --git a/lib/IR/ValueTypes.cpp b/lib/IR/ValueTypes.cpp index f2932302af2a4..ff1e431c2e9a0 100644 --- a/lib/IR/ValueTypes.cpp +++ b/lib/IR/ValueTypes.cpp @@ -55,6 +55,11 @@ bool EVT::isExtendedInteger() const { return LLVMTy->isIntOrIntVectorTy(); } +bool EVT::isExtendedScalarInteger() const { + assert(isExtended() && "Type is not extended!"); + return LLVMTy->isIntegerTy(); +} + bool EVT::isExtendedVector() const { assert(isExtended() && "Type is not extended!"); return LLVMTy->isVectorTy(); diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp index 9198b0e1fb587..682e934d4b0f5 100644 --- a/lib/IR/Verifier.cpp +++ b/lib/IR/Verifier.cpp @@ -59,6 +59,7 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/InstIterator.h" @@ -67,6 +68,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/IR/ModuleSlotTracker.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Statepoint.h" #include "llvm/Pass.h" @@ -82,14 +84,18 @@ static cl::opt<bool> VerifyDebugInfo("verify-debug-info", cl::init(true)); namespace { struct VerifierSupport { - raw_ostream &OS; - const Module *M; + raw_ostream *OS; + const Module *M = nullptr; + Optional<ModuleSlotTracker> MST; - /// \brief Track the brokenness of the module while recursively visiting. - bool Broken; + /// Track the brokenness of the module while recursively visiting. + bool Broken = false; + /// Broken debug info can be "recovered" from by stripping the debug info. + bool BrokenDebugInfo = false; + /// Whether to treat broken debug info as an error. + bool TreatBrokenDebugInfoAsError = true; - explicit VerifierSupport(raw_ostream &OS) - : OS(OS), M(nullptr), Broken(false) {} + explicit VerifierSupport(raw_ostream *OS) : OS(OS) {} private: template <class NodeTy> void Write(const ilist_iterator<NodeTy> &I) { @@ -99,17 +105,18 @@ private: void Write(const Module *M) { if (!M) return; - OS << "; ModuleID = '" << M->getModuleIdentifier() << "'\n"; + *OS << "; ModuleID = '" << M->getModuleIdentifier() << "'\n"; } void Write(const Value *V) { if (!V) return; if (isa<Instruction>(V)) { - OS << *V << '\n'; + V->print(*OS, *MST); + *OS << '\n'; } else { - V->printAsOperand(OS, true, M); - OS << '\n'; + V->printAsOperand(*OS, true, *MST); + *OS << '\n'; } } void Write(ImmutableCallSite CS) { @@ -119,8 +126,8 @@ private: void Write(const Metadata *MD) { if (!MD) return; - MD->print(OS, M); - OS << '\n'; + MD->print(*OS, *MST, M); + *OS << '\n'; } template <class T> void Write(const MDTupleTypedArrayWrapper<T> &MD) { @@ -130,20 +137,20 @@ private: void Write(const NamedMDNode *NMD) { if (!NMD) return; - NMD->print(OS); - OS << '\n'; + NMD->print(*OS, *MST); + *OS << '\n'; } void Write(Type *T) { if (!T) return; - OS << ' ' << *T; + *OS << ' ' << *T; } void Write(const Comdat *C) { if (!C) return; - OS << *C; + *OS << *C; } template <typename T> void Write(ArrayRef<T> Vs) { @@ -165,7 +172,8 @@ public: /// This provides a nice place to put a breakpoint if you want to see why /// something is not correct. void CheckFailed(const Twine &Message) { - OS << Message << '\n'; + if (OS) + *OS << Message << '\n'; Broken = true; } @@ -176,7 +184,25 @@ public: template <typename T1, typename... Ts> void CheckFailed(const Twine &Message, const T1 &V1, const Ts &... Vs) { CheckFailed(Message); - WriteTs(V1, Vs...); + if (OS) + WriteTs(V1, Vs...); + } + + /// A debug info check failed. + void DebugInfoCheckFailed(const Twine &Message) { + if (OS) + *OS << Message << '\n'; + Broken |= TreatBrokenDebugInfoAsError; + BrokenDebugInfo = true; + } + + /// A debug info check failed (with values to print). + template <typename T1, typename... Ts> + void DebugInfoCheckFailed(const Twine &Message, const T1 &V1, + const Ts &... Vs) { + DebugInfoCheckFailed(Message); + if (OS) + WriteTs(V1, Vs...); } }; @@ -196,8 +222,8 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport { /// \brief Keep track of the metadata nodes that have been checked already. SmallPtrSet<const Metadata *, 32> MDNodes; - /// \brief Track unresolved string-based type references. - SmallDenseMap<const MDString *, const MDNode *, 32> UnresolvedTypeRefs; + /// Track all DICompileUnits visited. + SmallPtrSet<const Metadata *, 2> CUVisited; /// \brief The result type for a landingpad. Type *LandingPadResultTy; @@ -217,41 +243,60 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport { /// Cache of constants visited in search of ConstantExprs. SmallPtrSet<const Constant *, 32> ConstantExprVisited; + /// Cache of declarations of the llvm.experimental.deoptimize.<ty> intrinsic. + SmallVector<const Function *, 4> DeoptimizeDeclarations; + + // Verify that this GlobalValue is only used in this module. + // This map is used to avoid visiting uses twice. We can arrive at a user + // twice, if they have multiple operands. In particular for very large + // constant expressions, we can arrive at a particular user many times. + SmallPtrSet<const Value *, 32> GlobalValueVisited; + void checkAtomicMemAccessSize(const Module *M, Type *Ty, const Instruction *I); + + void updateModule(const Module *NewM) { + if (M == NewM) + return; + MST.emplace(NewM); + M = NewM; + } + public: - explicit Verifier(raw_ostream &OS) + explicit Verifier(raw_ostream *OS, bool ShouldTreatBrokenDebugInfoAsError) : VerifierSupport(OS), Context(nullptr), LandingPadResultTy(nullptr), - SawFrameEscape(false) {} + SawFrameEscape(false) { + TreatBrokenDebugInfoAsError = ShouldTreatBrokenDebugInfoAsError; + } + + bool hasBrokenDebugInfo() const { return BrokenDebugInfo; } bool verify(const Function &F) { - M = F.getParent(); + updateModule(F.getParent()); Context = &M->getContext(); // First ensure the function is well-enough formed to compute dominance - // information. - if (F.empty()) { - OS << "Function '" << F.getName() - << "' does not contain an entry block!\n"; - return false; - } - for (Function::const_iterator I = F.begin(), E = F.end(); I != E; ++I) { - if (I->empty() || !I->back().isTerminator()) { - OS << "Basic Block in function '" << F.getName() - << "' does not have terminator!\n"; - I->printAsOperand(OS, true); - OS << "\n"; - return false; + // information, and directly compute a dominance tree. We don't rely on the + // pass manager to provide this as it isolates us from a potentially + // out-of-date dominator tree and makes it significantly more complex to run + // this code outside of a pass manager. + // FIXME: It's really gross that we have to cast away constness here. + if (!F.empty()) + DT.recalculate(const_cast<Function &>(F)); + + for (const BasicBlock &BB : F) { + if (!BB.empty() && BB.back().isTerminator()) + continue; + + if (OS) { + *OS << "Basic Block in function '" << F.getName() + << "' does not have terminator!\n"; + BB.printAsOperand(*OS, true, *MST); + *OS << "\n"; } + return false; } - // Now directly compute a dominance tree. We don't rely on the pass - // manager to provide this as it isolates us from a potentially - // out-of-date dominator tree and makes it significantly more complex to - // run this code outside of a pass manager. - // FIXME: It's really gross that we have to cast away constness here. - DT.recalculate(const_cast<Function &>(F)); - Broken = false; // FIXME: We strip const here because the inst visitor strips const. visit(const_cast<Function &>(F)); @@ -265,35 +310,26 @@ public: } bool verify(const Module &M) { - this->M = &M; + updateModule(&M); Context = &M.getContext(); Broken = false; - // Scan through, checking all of the external function's linkage now... - for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I) { - visitGlobalValue(*I); - - // Check to make sure function prototypes are okay. - if (I->isDeclaration()) - visitFunction(*I); - } + // Collect all declarations of the llvm.experimental.deoptimize intrinsic. + for (const Function &F : M) + if (F.getIntrinsicID() == Intrinsic::experimental_deoptimize) + DeoptimizeDeclarations.push_back(&F); // Now that we've visited every function, verify that we never asked to // recover a frame index that wasn't escaped. verifyFrameRecoverIndices(); + for (const GlobalVariable &GV : M.globals()) + visitGlobalVariable(GV); - for (Module::const_global_iterator I = M.global_begin(), E = M.global_end(); - I != E; ++I) - visitGlobalVariable(*I); + for (const GlobalAlias &GA : M.aliases()) + visitGlobalAlias(GA); - for (Module::const_alias_iterator I = M.alias_begin(), E = M.alias_end(); - I != E; ++I) - visitGlobalAlias(*I); - - for (Module::const_named_metadata_iterator I = M.named_metadata_begin(), - E = M.named_metadata_end(); - I != E; ++I) - visitNamedMDNode(*I); + for (const NamedMDNode &NMD : M.named_metadata()) + visitNamedMDNode(NMD); for (const StringMapEntry<Comdat> &SMEC : M.getComdatSymbolTable()) visitComdat(SMEC.getValue()); @@ -301,8 +337,9 @@ public: visitModuleFlags(M); visitModuleIdents(M); - // Verify type referneces last. - verifyTypeRefs(); + verifyCompileUnits(); + + verifyDeoptimizeCallingConvs(); return !Broken; } @@ -340,27 +377,6 @@ private: void visitTemplateParams(const MDNode &N, const Metadata &RawParams); - /// \brief Check for a valid string-based type reference. - /// - /// Checks if \c MD is a string-based type reference. If it is, keeps track - /// of it (and its user, \c N) for error messages later. - bool isValidUUID(const MDNode &N, const Metadata *MD); - - /// \brief Check for a valid type reference. - /// - /// Checks for subclasses of \a DIType, or \a isValidUUID(). - bool isTypeRef(const MDNode &N, const Metadata *MD); - - /// \brief Check for a valid scope reference. - /// - /// Checks for subclasses of \a DIScope, or \a isValidUUID(). - bool isScopeRef(const MDNode &N, const Metadata *MD); - - /// \brief Check for a valid debug info reference. - /// - /// Checks for subclasses of \a DINode, or \a isValidUUID(). - bool isDIRef(const MDNode &N, const Metadata *MD); - // InstVisitor overrides... using InstVisitor<Verifier>::visit; void visit(Instruction &I); @@ -419,53 +435,69 @@ private: void visitCatchSwitchInst(CatchSwitchInst &CatchSwitch); void visitCleanupReturnInst(CleanupReturnInst &CRI); - void VerifyCallSite(CallSite CS); + void verifyCallSite(CallSite CS); + void verifySwiftErrorCallSite(CallSite CS, const Value *SwiftErrorVal); + void verifySwiftErrorValue(const Value *SwiftErrorVal); void verifyMustTailCall(CallInst &CI); - bool PerformTypeCheck(Intrinsic::ID ID, Function *F, Type *Ty, int VT, + bool performTypeCheck(Intrinsic::ID ID, Function *F, Type *Ty, int VT, unsigned ArgNo, std::string &Suffix); - bool VerifyIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos, - SmallVectorImpl<Type *> &ArgTys); - bool VerifyIntrinsicIsVarArg(bool isVarArg, - ArrayRef<Intrinsic::IITDescriptor> &Infos); - bool VerifyAttributeCount(AttributeSet Attrs, unsigned Params); - void VerifyAttributeTypes(AttributeSet Attrs, unsigned Idx, bool isFunction, + bool verifyAttributeCount(AttributeSet Attrs, unsigned Params); + void verifyAttributeTypes(AttributeSet Attrs, unsigned Idx, bool isFunction, const Value *V); - void VerifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, + void verifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, bool isReturnValue, const Value *V); - void VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, + void verifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, const Value *V); - void VerifyFunctionMetadata( - const SmallVector<std::pair<unsigned, MDNode *>, 4> MDs); + void verifyFunctionMetadata(ArrayRef<std::pair<unsigned, MDNode *>> MDs); void visitConstantExprsRecursively(const Constant *EntryC); void visitConstantExpr(const ConstantExpr *CE); - void VerifyStatepoint(ImmutableCallSite CS); + void verifyStatepoint(ImmutableCallSite CS); void verifyFrameRecoverIndices(); void verifySiblingFuncletUnwinds(); - // Module-level debug info verification... - void verifyTypeRefs(); - template <class MapTy> - void verifyBitPieceExpression(const DbgInfoIntrinsic &I, - const MapTy &TypeRefs); - void visitUnresolvedTypeRef(const MDString *S, const MDNode *N); + void verifyBitPieceExpression(const DbgInfoIntrinsic &I); + + /// Module-level debug info verification... + void verifyCompileUnits(); + + /// Module-level verification that all @llvm.experimental.deoptimize + /// declarations share the same calling convention. + void verifyDeoptimizeCallingConvs(); }; } // End anonymous namespace -// Assert - We know that cond should be true, if not print an error message. +/// We know that cond should be true, if not print an error message. #define Assert(C, ...) \ do { if (!(C)) { CheckFailed(__VA_ARGS__); return; } } while (0) +/// We know that a debug info condition should be true, if not print +/// an error message. +#define AssertDI(C, ...) \ + do { if (!(C)) { DebugInfoCheckFailed(__VA_ARGS__); return; } } while (0) + + void Verifier::visit(Instruction &I) { for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) Assert(I.getOperand(i) != nullptr, "Operand is null", &I); InstVisitor<Verifier>::visit(I); } +// Helper to recursively iterate over indirect users. By +// returning false, the callback can ask to stop recursing +// further. +static void forEachUser(const Value *User, + SmallPtrSet<const Value *, 32> &Visited, + llvm::function_ref<bool(const Value *)> Callback) { + if (!Visited.insert(User).second) + return; + for (const Value *TheNextUser : User->materialized_users()) + if (Callback(TheNextUser)) + forEachUser(TheNextUser, Visited, Callback); +} void Verifier::visitGlobalValue(const GlobalValue &GV) { - Assert(!GV.isDeclaration() || GV.hasExternalLinkage() || - GV.hasExternalWeakLinkage(), + Assert(!GV.isDeclaration() || GV.hasValidDeclarationLinkage(), "Global is external, but doesn't have external or weak linkage!", &GV); Assert(GV.getAlignment() <= Value::MaximumAlignment, @@ -481,11 +513,30 @@ void Verifier::visitGlobalValue(const GlobalValue &GV) { if (GV.isDeclarationForLinker()) Assert(!GV.hasComdat(), "Declaration may not be in a Comdat!", &GV); + + forEachUser(&GV, GlobalValueVisited, [&](const Value *V) -> bool { + if (const Instruction *I = dyn_cast<Instruction>(V)) { + if (!I->getParent() || !I->getParent()->getParent()) + CheckFailed("Global is referenced by parentless instruction!", &GV, + M, I); + else if (I->getParent()->getParent()->getParent() != M) + CheckFailed("Global is referenced in a different module!", &GV, + M, I, I->getParent()->getParent(), + I->getParent()->getParent()->getParent()); + return false; + } else if (const Function *F = dyn_cast<Function>(V)) { + if (F->getParent() != M) + CheckFailed("Global is used by function in a different module", &GV, + M, F, F->getParent()); + return false; + } + return true; + }); } void Verifier::visitGlobalVariable(const GlobalVariable &GV) { if (GV.hasInitializer()) { - Assert(GV.getInitializer()->getType() == GV.getType()->getElementType(), + Assert(GV.getInitializer()->getType() == GV.getValueType(), "Global variable initializer type does not match global " "variable type!", &GV); @@ -499,9 +550,6 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { &GV); Assert(!GV.hasComdat(), "'common' global may not be in a Comdat!", &GV); } - } else { - Assert(GV.hasExternalLinkage() || GV.hasExternalWeakLinkage(), - "invalid linkage type for global declaration", &GV); } if (GV.hasName() && (GV.getName() == "llvm.global_ctors" || @@ -542,8 +590,8 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) { const ConstantArray *InitArray = dyn_cast<ConstantArray>(Init); Assert(InitArray, "wrong initalizer for intrinsic global variable", Init); - for (unsigned i = 0, e = InitArray->getNumOperands(); i != e; ++i) { - Value *V = Init->getOperand(i)->stripPointerCastsNoFollowAliases(); + for (Value *Op : InitArray->operands()) { + Value *V = Op->stripPointerCastsNoFollowAliases(); Assert(isa<GlobalVariable>(V) || isa<Function>(V) || isa<GlobalAlias>(V), "invalid llvm.used member", V); @@ -584,7 +632,7 @@ void Verifier::visitAliaseeSubExpr(SmallPtrSetImpl<const GlobalAlias*> &Visited, if (const auto *GA2 = dyn_cast<GlobalAlias>(GV)) { Assert(Visited.insert(GA2).second, "Aliases cannot form a cycle", &GA); - Assert(!GA2->mayBeOverridden(), "Alias cannot point to a weak alias", + Assert(!GA2->isInterposable(), "Alias cannot point to an interposable alias", &GA); } else { // Only continue verifying subexpressions of GlobalAliases. @@ -624,11 +672,9 @@ void Verifier::visitGlobalAlias(const GlobalAlias &GA) { } void Verifier::visitNamedMDNode(const NamedMDNode &NMD) { - for (unsigned i = 0, e = NMD.getNumOperands(); i != e; ++i) { - MDNode *MD = NMD.getOperand(i); - + for (const MDNode *MD : NMD.operands()) { if (NMD.getName() == "llvm.dbg.cu") { - Assert(MD && isa<DICompileUnit>(MD), "invalid compile unit", &NMD, MD); + AssertDI(MD && isa<DICompileUnit>(MD), "invalid compile unit", &NMD, MD); } if (!MD) @@ -656,8 +702,7 @@ void Verifier::visitMDNode(const MDNode &MD) { #include "llvm/IR/Metadata.def" } - for (unsigned i = 0, e = MD.getNumOperands(); i != e; ++i) { - Metadata *Op = MD.getOperand(i); + for (const Metadata *Op : MD.operands()) { if (!Op) continue; Assert(!isa<LocalAsMetadata>(Op), "Invalid operand for global metadata!", @@ -719,33 +764,9 @@ void Verifier::visitMetadataAsValue(const MetadataAsValue &MDV, Function *F) { visitValueAsMetadata(*V, F); } -bool Verifier::isValidUUID(const MDNode &N, const Metadata *MD) { - auto *S = dyn_cast<MDString>(MD); - if (!S) - return false; - if (S->getString().empty()) - return false; - - // Keep track of names of types referenced via UUID so we can check that they - // actually exist. - UnresolvedTypeRefs.insert(std::make_pair(S, &N)); - return true; -} - -/// \brief Check if a value can be a reference to a type. -bool Verifier::isTypeRef(const MDNode &N, const Metadata *MD) { - return !MD || isValidUUID(N, MD) || isa<DIType>(MD); -} - -/// \brief Check if a value can be a ScopeRef. -bool Verifier::isScopeRef(const MDNode &N, const Metadata *MD) { - return !MD || isValidUUID(N, MD) || isa<DIScope>(MD); -} - -/// \brief Check if a value can be a debug info ref. -bool Verifier::isDIRef(const MDNode &N, const Metadata *MD) { - return !MD || isValidUUID(N, MD) || isa<DINode>(MD); -} +static bool isType(const Metadata *MD) { return !MD || isa<DIType>(MD); } +static bool isScope(const Metadata *MD) { return !MD || isa<DIScope>(MD); } +static bool isDINode(const Metadata *MD) { return !MD || isa<DINode>(MD); } template <class Ty> bool isValidMetadataArrayImpl(const MDTuple &N, bool AllowNull) { @@ -772,60 +793,60 @@ bool isValidMetadataNullArray(const MDTuple &N) { } void Verifier::visitDILocation(const DILocation &N) { - Assert(N.getRawScope() && isa<DILocalScope>(N.getRawScope()), - "location requires a valid scope", &N, N.getRawScope()); + AssertDI(N.getRawScope() && isa<DILocalScope>(N.getRawScope()), + "location requires a valid scope", &N, N.getRawScope()); if (auto *IA = N.getRawInlinedAt()) - Assert(isa<DILocation>(IA), "inlined-at should be a location", &N, IA); + AssertDI(isa<DILocation>(IA), "inlined-at should be a location", &N, IA); } void Verifier::visitGenericDINode(const GenericDINode &N) { - Assert(N.getTag(), "invalid tag", &N); + AssertDI(N.getTag(), "invalid tag", &N); } void Verifier::visitDIScope(const DIScope &N) { if (auto *F = N.getRawFile()) - Assert(isa<DIFile>(F), "invalid file", &N, F); + AssertDI(isa<DIFile>(F), "invalid file", &N, F); } void Verifier::visitDISubrange(const DISubrange &N) { - Assert(N.getTag() == dwarf::DW_TAG_subrange_type, "invalid tag", &N); - Assert(N.getCount() >= -1, "invalid subrange count", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_subrange_type, "invalid tag", &N); + AssertDI(N.getCount() >= -1, "invalid subrange count", &N); } void Verifier::visitDIEnumerator(const DIEnumerator &N) { - Assert(N.getTag() == dwarf::DW_TAG_enumerator, "invalid tag", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_enumerator, "invalid tag", &N); } void Verifier::visitDIBasicType(const DIBasicType &N) { - Assert(N.getTag() == dwarf::DW_TAG_base_type || - N.getTag() == dwarf::DW_TAG_unspecified_type, - "invalid tag", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_base_type || + N.getTag() == dwarf::DW_TAG_unspecified_type, + "invalid tag", &N); } void Verifier::visitDIDerivedType(const DIDerivedType &N) { // Common scope checks. visitDIScope(N); - Assert(N.getTag() == dwarf::DW_TAG_typedef || - N.getTag() == dwarf::DW_TAG_pointer_type || - N.getTag() == dwarf::DW_TAG_ptr_to_member_type || - N.getTag() == dwarf::DW_TAG_reference_type || - N.getTag() == dwarf::DW_TAG_rvalue_reference_type || - N.getTag() == dwarf::DW_TAG_const_type || - N.getTag() == dwarf::DW_TAG_volatile_type || - N.getTag() == dwarf::DW_TAG_restrict_type || - N.getTag() == dwarf::DW_TAG_member || - N.getTag() == dwarf::DW_TAG_inheritance || - N.getTag() == dwarf::DW_TAG_friend, - "invalid tag", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_typedef || + N.getTag() == dwarf::DW_TAG_pointer_type || + N.getTag() == dwarf::DW_TAG_ptr_to_member_type || + N.getTag() == dwarf::DW_TAG_reference_type || + N.getTag() == dwarf::DW_TAG_rvalue_reference_type || + N.getTag() == dwarf::DW_TAG_const_type || + N.getTag() == dwarf::DW_TAG_volatile_type || + N.getTag() == dwarf::DW_TAG_restrict_type || + N.getTag() == dwarf::DW_TAG_member || + N.getTag() == dwarf::DW_TAG_inheritance || + N.getTag() == dwarf::DW_TAG_friend, + "invalid tag", &N); if (N.getTag() == dwarf::DW_TAG_ptr_to_member_type) { - Assert(isTypeRef(N, N.getExtraData()), "invalid pointer to member type", &N, - N.getExtraData()); + AssertDI(isType(N.getRawExtraData()), "invalid pointer to member type", &N, + N.getRawExtraData()); } - Assert(isScopeRef(N, N.getScope()), "invalid scope", &N, N.getScope()); - Assert(isTypeRef(N, N.getBaseType()), "invalid base type", &N, - N.getBaseType()); + AssertDI(isScope(N.getRawScope()), "invalid scope", &N, N.getRawScope()); + AssertDI(isType(N.getRawBaseType()), "invalid base type", &N, + N.getRawBaseType()); } static bool hasConflictingReferenceFlags(unsigned Flags) { @@ -835,10 +856,10 @@ static bool hasConflictingReferenceFlags(unsigned Flags) { void Verifier::visitTemplateParams(const MDNode &N, const Metadata &RawParams) { auto *Params = dyn_cast<MDTuple>(&RawParams); - Assert(Params, "invalid template params", &N, &RawParams); + AssertDI(Params, "invalid template params", &N, &RawParams); for (Metadata *Op : Params->operands()) { - Assert(Op && isa<DITemplateParameter>(Op), "invalid template parameter", &N, - Params, Op); + AssertDI(Op && isa<DITemplateParameter>(Op), "invalid template parameter", + &N, Params, Op); } } @@ -846,141 +867,151 @@ void Verifier::visitDICompositeType(const DICompositeType &N) { // Common scope checks. visitDIScope(N); - Assert(N.getTag() == dwarf::DW_TAG_array_type || - N.getTag() == dwarf::DW_TAG_structure_type || - N.getTag() == dwarf::DW_TAG_union_type || - N.getTag() == dwarf::DW_TAG_enumeration_type || - N.getTag() == dwarf::DW_TAG_class_type, - "invalid tag", &N); - - Assert(isScopeRef(N, N.getScope()), "invalid scope", &N, N.getScope()); - Assert(isTypeRef(N, N.getBaseType()), "invalid base type", &N, - N.getBaseType()); - - Assert(!N.getRawElements() || isa<MDTuple>(N.getRawElements()), - "invalid composite elements", &N, N.getRawElements()); - Assert(isTypeRef(N, N.getRawVTableHolder()), "invalid vtable holder", &N, - N.getRawVTableHolder()); - Assert(!hasConflictingReferenceFlags(N.getFlags()), "invalid reference flags", - &N); + AssertDI(N.getTag() == dwarf::DW_TAG_array_type || + N.getTag() == dwarf::DW_TAG_structure_type || + N.getTag() == dwarf::DW_TAG_union_type || + N.getTag() == dwarf::DW_TAG_enumeration_type || + N.getTag() == dwarf::DW_TAG_class_type, + "invalid tag", &N); + + AssertDI(isScope(N.getRawScope()), "invalid scope", &N, N.getRawScope()); + AssertDI(isType(N.getRawBaseType()), "invalid base type", &N, + N.getRawBaseType()); + + AssertDI(!N.getRawElements() || isa<MDTuple>(N.getRawElements()), + "invalid composite elements", &N, N.getRawElements()); + AssertDI(isType(N.getRawVTableHolder()), "invalid vtable holder", &N, + N.getRawVTableHolder()); + AssertDI(!hasConflictingReferenceFlags(N.getFlags()), + "invalid reference flags", &N); if (auto *Params = N.getRawTemplateParams()) visitTemplateParams(N, *Params); if (N.getTag() == dwarf::DW_TAG_class_type || N.getTag() == dwarf::DW_TAG_union_type) { - Assert(N.getFile() && !N.getFile()->getFilename().empty(), - "class/union requires a filename", &N, N.getFile()); + AssertDI(N.getFile() && !N.getFile()->getFilename().empty(), + "class/union requires a filename", &N, N.getFile()); } } void Verifier::visitDISubroutineType(const DISubroutineType &N) { - Assert(N.getTag() == dwarf::DW_TAG_subroutine_type, "invalid tag", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_subroutine_type, "invalid tag", &N); if (auto *Types = N.getRawTypeArray()) { - Assert(isa<MDTuple>(Types), "invalid composite elements", &N, Types); + AssertDI(isa<MDTuple>(Types), "invalid composite elements", &N, Types); for (Metadata *Ty : N.getTypeArray()->operands()) { - Assert(isTypeRef(N, Ty), "invalid subroutine type ref", &N, Types, Ty); + AssertDI(isType(Ty), "invalid subroutine type ref", &N, Types, Ty); } } - Assert(!hasConflictingReferenceFlags(N.getFlags()), "invalid reference flags", - &N); + AssertDI(!hasConflictingReferenceFlags(N.getFlags()), + "invalid reference flags", &N); } void Verifier::visitDIFile(const DIFile &N) { - Assert(N.getTag() == dwarf::DW_TAG_file_type, "invalid tag", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_file_type, "invalid tag", &N); } void Verifier::visitDICompileUnit(const DICompileUnit &N) { - Assert(N.isDistinct(), "compile units must be distinct", &N); - Assert(N.getTag() == dwarf::DW_TAG_compile_unit, "invalid tag", &N); + AssertDI(N.isDistinct(), "compile units must be distinct", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_compile_unit, "invalid tag", &N); // Don't bother verifying the compilation directory or producer string // as those could be empty. - Assert(N.getRawFile() && isa<DIFile>(N.getRawFile()), "invalid file", &N, - N.getRawFile()); - Assert(!N.getFile()->getFilename().empty(), "invalid filename", &N, - N.getFile()); + AssertDI(N.getRawFile() && isa<DIFile>(N.getRawFile()), "invalid file", &N, + N.getRawFile()); + AssertDI(!N.getFile()->getFilename().empty(), "invalid filename", &N, + N.getFile()); + + AssertDI((N.getEmissionKind() <= DICompileUnit::LastEmissionKind), + "invalid emission kind", &N); if (auto *Array = N.getRawEnumTypes()) { - Assert(isa<MDTuple>(Array), "invalid enum list", &N, Array); + AssertDI(isa<MDTuple>(Array), "invalid enum list", &N, Array); for (Metadata *Op : N.getEnumTypes()->operands()) { auto *Enum = dyn_cast_or_null<DICompositeType>(Op); - Assert(Enum && Enum->getTag() == dwarf::DW_TAG_enumeration_type, - "invalid enum type", &N, N.getEnumTypes(), Op); + AssertDI(Enum && Enum->getTag() == dwarf::DW_TAG_enumeration_type, + "invalid enum type", &N, N.getEnumTypes(), Op); } } if (auto *Array = N.getRawRetainedTypes()) { - Assert(isa<MDTuple>(Array), "invalid retained type list", &N, Array); + AssertDI(isa<MDTuple>(Array), "invalid retained type list", &N, Array); for (Metadata *Op : N.getRetainedTypes()->operands()) { - Assert(Op && isa<DIType>(Op), "invalid retained type", &N, Op); - } - } - if (auto *Array = N.getRawSubprograms()) { - Assert(isa<MDTuple>(Array), "invalid subprogram list", &N, Array); - for (Metadata *Op : N.getSubprograms()->operands()) { - Assert(Op && isa<DISubprogram>(Op), "invalid subprogram ref", &N, Op); + AssertDI(Op && (isa<DIType>(Op) || + (isa<DISubprogram>(Op) && + cast<DISubprogram>(Op)->isDefinition() == false)), + "invalid retained type", &N, Op); } } if (auto *Array = N.getRawGlobalVariables()) { - Assert(isa<MDTuple>(Array), "invalid global variable list", &N, Array); + AssertDI(isa<MDTuple>(Array), "invalid global variable list", &N, Array); for (Metadata *Op : N.getGlobalVariables()->operands()) { - Assert(Op && isa<DIGlobalVariable>(Op), "invalid global variable ref", &N, - Op); + AssertDI(Op && isa<DIGlobalVariable>(Op), "invalid global variable ref", + &N, Op); } } if (auto *Array = N.getRawImportedEntities()) { - Assert(isa<MDTuple>(Array), "invalid imported entity list", &N, Array); + AssertDI(isa<MDTuple>(Array), "invalid imported entity list", &N, Array); for (Metadata *Op : N.getImportedEntities()->operands()) { - Assert(Op && isa<DIImportedEntity>(Op), "invalid imported entity ref", &N, - Op); + AssertDI(Op && isa<DIImportedEntity>(Op), "invalid imported entity ref", + &N, Op); } } if (auto *Array = N.getRawMacros()) { - Assert(isa<MDTuple>(Array), "invalid macro list", &N, Array); + AssertDI(isa<MDTuple>(Array), "invalid macro list", &N, Array); for (Metadata *Op : N.getMacros()->operands()) { - Assert(Op && isa<DIMacroNode>(Op), "invalid macro ref", &N, Op); + AssertDI(Op && isa<DIMacroNode>(Op), "invalid macro ref", &N, Op); } } + CUVisited.insert(&N); } void Verifier::visitDISubprogram(const DISubprogram &N) { - Assert(N.getTag() == dwarf::DW_TAG_subprogram, "invalid tag", &N); - Assert(isScopeRef(N, N.getRawScope()), "invalid scope", &N, N.getRawScope()); + AssertDI(N.getTag() == dwarf::DW_TAG_subprogram, "invalid tag", &N); + AssertDI(isScope(N.getRawScope()), "invalid scope", &N, N.getRawScope()); + if (auto *F = N.getRawFile()) + AssertDI(isa<DIFile>(F), "invalid file", &N, F); if (auto *T = N.getRawType()) - Assert(isa<DISubroutineType>(T), "invalid subroutine type", &N, T); - Assert(isTypeRef(N, N.getRawContainingType()), "invalid containing type", &N, - N.getRawContainingType()); + AssertDI(isa<DISubroutineType>(T), "invalid subroutine type", &N, T); + AssertDI(isType(N.getRawContainingType()), "invalid containing type", &N, + N.getRawContainingType()); if (auto *Params = N.getRawTemplateParams()) visitTemplateParams(N, *Params); - if (auto *S = N.getRawDeclaration()) { - Assert(isa<DISubprogram>(S) && !cast<DISubprogram>(S)->isDefinition(), - "invalid subprogram declaration", &N, S); - } + if (auto *S = N.getRawDeclaration()) + AssertDI(isa<DISubprogram>(S) && !cast<DISubprogram>(S)->isDefinition(), + "invalid subprogram declaration", &N, S); if (auto *RawVars = N.getRawVariables()) { auto *Vars = dyn_cast<MDTuple>(RawVars); - Assert(Vars, "invalid variable list", &N, RawVars); + AssertDI(Vars, "invalid variable list", &N, RawVars); for (Metadata *Op : Vars->operands()) { - Assert(Op && isa<DILocalVariable>(Op), "invalid local variable", &N, Vars, - Op); + AssertDI(Op && isa<DILocalVariable>(Op), "invalid local variable", &N, + Vars, Op); } } - Assert(!hasConflictingReferenceFlags(N.getFlags()), "invalid reference flags", - &N); - - if (N.isDefinition()) - Assert(N.isDistinct(), "subprogram definitions must be distinct", &N); + AssertDI(!hasConflictingReferenceFlags(N.getFlags()), + "invalid reference flags", &N); + + auto *Unit = N.getRawUnit(); + if (N.isDefinition()) { + // Subprogram definitions (not part of the type hierarchy). + AssertDI(N.isDistinct(), "subprogram definitions must be distinct", &N); + AssertDI(Unit, "subprogram definitions must have a compile unit", &N); + AssertDI(isa<DICompileUnit>(Unit), "invalid unit type", &N, Unit); + } else { + // Subprogram declarations (part of the type hierarchy). + AssertDI(!Unit, "subprogram declarations must not have a compile unit", &N); + } } void Verifier::visitDILexicalBlockBase(const DILexicalBlockBase &N) { - Assert(N.getTag() == dwarf::DW_TAG_lexical_block, "invalid tag", &N); - Assert(N.getRawScope() && isa<DILocalScope>(N.getRawScope()), - "invalid local scope", &N, N.getRawScope()); + AssertDI(N.getTag() == dwarf::DW_TAG_lexical_block, "invalid tag", &N); + AssertDI(N.getRawScope() && isa<DILocalScope>(N.getRawScope()), + "invalid local scope", &N, N.getRawScope()); } void Verifier::visitDILexicalBlock(const DILexicalBlock &N) { visitDILexicalBlockBase(N); - Assert(N.getLine() || !N.getColumn(), - "cannot have column info without line info", &N); + AssertDI(N.getLine() || !N.getColumn(), + "cannot have column info without line info", &N); } void Verifier::visitDILexicalBlockFile(const DILexicalBlockFile &N) { @@ -988,83 +1019,84 @@ void Verifier::visitDILexicalBlockFile(const DILexicalBlockFile &N) { } void Verifier::visitDINamespace(const DINamespace &N) { - Assert(N.getTag() == dwarf::DW_TAG_namespace, "invalid tag", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_namespace, "invalid tag", &N); if (auto *S = N.getRawScope()) - Assert(isa<DIScope>(S), "invalid scope ref", &N, S); + AssertDI(isa<DIScope>(S), "invalid scope ref", &N, S); } void Verifier::visitDIMacro(const DIMacro &N) { - Assert(N.getMacinfoType() == dwarf::DW_MACINFO_define || - N.getMacinfoType() == dwarf::DW_MACINFO_undef, - "invalid macinfo type", &N); - Assert(!N.getName().empty(), "anonymous macro", &N); + AssertDI(N.getMacinfoType() == dwarf::DW_MACINFO_define || + N.getMacinfoType() == dwarf::DW_MACINFO_undef, + "invalid macinfo type", &N); + AssertDI(!N.getName().empty(), "anonymous macro", &N); if (!N.getValue().empty()) { assert(N.getValue().data()[0] != ' ' && "Macro value has a space prefix"); } } void Verifier::visitDIMacroFile(const DIMacroFile &N) { - Assert(N.getMacinfoType() == dwarf::DW_MACINFO_start_file, - "invalid macinfo type", &N); + AssertDI(N.getMacinfoType() == dwarf::DW_MACINFO_start_file, + "invalid macinfo type", &N); if (auto *F = N.getRawFile()) - Assert(isa<DIFile>(F), "invalid file", &N, F); + AssertDI(isa<DIFile>(F), "invalid file", &N, F); if (auto *Array = N.getRawElements()) { - Assert(isa<MDTuple>(Array), "invalid macro list", &N, Array); + AssertDI(isa<MDTuple>(Array), "invalid macro list", &N, Array); for (Metadata *Op : N.getElements()->operands()) { - Assert(Op && isa<DIMacroNode>(Op), "invalid macro ref", &N, Op); + AssertDI(Op && isa<DIMacroNode>(Op), "invalid macro ref", &N, Op); } } } void Verifier::visitDIModule(const DIModule &N) { - Assert(N.getTag() == dwarf::DW_TAG_module, "invalid tag", &N); - Assert(!N.getName().empty(), "anonymous module", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_module, "invalid tag", &N); + AssertDI(!N.getName().empty(), "anonymous module", &N); } void Verifier::visitDITemplateParameter(const DITemplateParameter &N) { - Assert(isTypeRef(N, N.getType()), "invalid type ref", &N, N.getType()); + AssertDI(isType(N.getRawType()), "invalid type ref", &N, N.getRawType()); } void Verifier::visitDITemplateTypeParameter(const DITemplateTypeParameter &N) { visitDITemplateParameter(N); - Assert(N.getTag() == dwarf::DW_TAG_template_type_parameter, "invalid tag", - &N); + AssertDI(N.getTag() == dwarf::DW_TAG_template_type_parameter, "invalid tag", + &N); } void Verifier::visitDITemplateValueParameter( const DITemplateValueParameter &N) { visitDITemplateParameter(N); - Assert(N.getTag() == dwarf::DW_TAG_template_value_parameter || - N.getTag() == dwarf::DW_TAG_GNU_template_template_param || - N.getTag() == dwarf::DW_TAG_GNU_template_parameter_pack, - "invalid tag", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_template_value_parameter || + N.getTag() == dwarf::DW_TAG_GNU_template_template_param || + N.getTag() == dwarf::DW_TAG_GNU_template_parameter_pack, + "invalid tag", &N); } void Verifier::visitDIVariable(const DIVariable &N) { if (auto *S = N.getRawScope()) - Assert(isa<DIScope>(S), "invalid scope", &N, S); - Assert(isTypeRef(N, N.getRawType()), "invalid type ref", &N, N.getRawType()); + AssertDI(isa<DIScope>(S), "invalid scope", &N, S); + AssertDI(isType(N.getRawType()), "invalid type ref", &N, N.getRawType()); if (auto *F = N.getRawFile()) - Assert(isa<DIFile>(F), "invalid file", &N, F); + AssertDI(isa<DIFile>(F), "invalid file", &N, F); } void Verifier::visitDIGlobalVariable(const DIGlobalVariable &N) { // Checks common to all variables. visitDIVariable(N); - Assert(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); - Assert(!N.getName().empty(), "missing global variable name", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); + AssertDI(!N.getName().empty(), "missing global variable name", &N); if (auto *V = N.getRawVariable()) { - Assert(isa<ConstantAsMetadata>(V) && - !isa<Function>(cast<ConstantAsMetadata>(V)->getValue()), - "invalid global varaible ref", &N, V); + AssertDI(isa<ConstantAsMetadata>(V) && + !isa<Function>(cast<ConstantAsMetadata>(V)->getValue()), + "invalid global varaible ref", &N, V); + visitConstantExprsRecursively(cast<ConstantAsMetadata>(V)->getValue()); } if (auto *Member = N.getRawStaticDataMemberDeclaration()) { - Assert(isa<DIDerivedType>(Member), "invalid static data member declaration", - &N, Member); + AssertDI(isa<DIDerivedType>(Member), + "invalid static data member declaration", &N, Member); } } @@ -1072,31 +1104,31 @@ void Verifier::visitDILocalVariable(const DILocalVariable &N) { // Checks common to all variables. visitDIVariable(N); - Assert(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); - Assert(N.getRawScope() && isa<DILocalScope>(N.getRawScope()), - "local variable requires a valid scope", &N, N.getRawScope()); + AssertDI(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N); + AssertDI(N.getRawScope() && isa<DILocalScope>(N.getRawScope()), + "local variable requires a valid scope", &N, N.getRawScope()); } void Verifier::visitDIExpression(const DIExpression &N) { - Assert(N.isValid(), "invalid expression", &N); + AssertDI(N.isValid(), "invalid expression", &N); } void Verifier::visitDIObjCProperty(const DIObjCProperty &N) { - Assert(N.getTag() == dwarf::DW_TAG_APPLE_property, "invalid tag", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_APPLE_property, "invalid tag", &N); if (auto *T = N.getRawType()) - Assert(isTypeRef(N, T), "invalid type ref", &N, T); + AssertDI(isType(T), "invalid type ref", &N, T); if (auto *F = N.getRawFile()) - Assert(isa<DIFile>(F), "invalid file", &N, F); + AssertDI(isa<DIFile>(F), "invalid file", &N, F); } void Verifier::visitDIImportedEntity(const DIImportedEntity &N) { - Assert(N.getTag() == dwarf::DW_TAG_imported_module || - N.getTag() == dwarf::DW_TAG_imported_declaration, - "invalid tag", &N); + AssertDI(N.getTag() == dwarf::DW_TAG_imported_module || + N.getTag() == dwarf::DW_TAG_imported_declaration, + "invalid tag", &N); if (auto *S = N.getRawScope()) - Assert(isa<DIScope>(S), "invalid scope for imported entity", &N, S); - Assert(isDIRef(N, N.getEntity()), "invalid imported entity", &N, - N.getEntity()); + AssertDI(isa<DIScope>(S), "invalid scope for imported entity", &N, S); + AssertDI(isDINode(N.getRawEntity()), "invalid imported entity", &N, + N.getRawEntity()); } void Verifier::visitComdat(const Comdat &C) { @@ -1114,8 +1146,7 @@ void Verifier::visitModuleIdents(const Module &M) { // llvm.ident takes a list of metadata entry. Each entry has only one string. // Scan each llvm.ident entry and make sure that this requirement is met. - for (unsigned i = 0, e = Idents->getNumOperands(); i != e; ++i) { - const MDNode *N = Idents->getOperand(i); + for (const MDNode *N : Idents->operands()) { Assert(N->getNumOperands() == 1, "incorrect number of operands in llvm.ident metadata", N); Assert(dyn_cast_or_null<MDString>(N->getOperand(0)), @@ -1132,13 +1163,11 @@ void Verifier::visitModuleFlags(const Module &M) { // Scan each flag, and track the flags and requirements. DenseMap<const MDString*, const MDNode*> SeenIDs; SmallVector<const MDNode*, 16> Requirements; - for (unsigned I = 0, E = Flags->getNumOperands(); I != E; ++I) { - visitModuleFlag(Flags->getOperand(I), SeenIDs, Requirements); - } + for (const MDNode *MDN : Flags->operands()) + visitModuleFlag(MDN, SeenIDs, Requirements); // Validate that the requirements in the module are valid. - for (unsigned I = 0, E = Requirements.size(); I != E; ++I) { - const MDNode *Requirement = Requirements[I]; + for (const MDNode *Requirement : Requirements) { const MDString *Flag = cast<MDString>(Requirement->getOperand(0)); const Metadata *ReqValue = Requirement->getOperand(1); @@ -1225,7 +1254,7 @@ Verifier::visitModuleFlag(const MDNode *Op, } } -void Verifier::VerifyAttributeTypes(AttributeSet Attrs, unsigned Idx, +void Verifier::verifyAttributeTypes(AttributeSet Attrs, unsigned Idx, bool isFunction, const Value *V) { unsigned Slot = ~0U; for (unsigned I = 0, E = Attrs.getNumSlots(); I != E; ++I) @@ -1272,13 +1301,15 @@ void Verifier::VerifyAttributeTypes(AttributeSet Attrs, unsigned Idx, I->getKindAsEnum() == Attribute::ArgMemOnly || I->getKindAsEnum() == Attribute::NoRecurse || I->getKindAsEnum() == Attribute::InaccessibleMemOnly || - I->getKindAsEnum() == Attribute::InaccessibleMemOrArgMemOnly) { + I->getKindAsEnum() == Attribute::InaccessibleMemOrArgMemOnly || + I->getKindAsEnum() == Attribute::AllocSize) { if (!isFunction) { CheckFailed("Attribute '" + I->getAsString() + "' only applies to functions!", V); return; } } else if (I->getKindAsEnum() == Attribute::ReadOnly || + I->getKindAsEnum() == Attribute::WriteOnly || I->getKindAsEnum() == Attribute::ReadNone) { if (Idx == 0) { CheckFailed("Attribute '" + I->getAsString() + @@ -1295,12 +1326,12 @@ void Verifier::VerifyAttributeTypes(AttributeSet Attrs, unsigned Idx, // VerifyParameterAttrs - Check the given attributes for an argument or return // value of the specified type. The value V is printed in error messages. -void Verifier::VerifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, +void Verifier::verifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, bool isReturnValue, const Value *V) { if (!Attrs.hasAttributes(Idx)) return; - VerifyAttributeTypes(Attrs, Idx, false, V); + verifyAttributeTypes(Attrs, Idx, false, V); if (isReturnValue) Assert(!Attrs.hasAttribute(Idx, Attribute::ByVal) && @@ -1308,9 +1339,12 @@ void Verifier::VerifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, !Attrs.hasAttribute(Idx, Attribute::StructRet) && !Attrs.hasAttribute(Idx, Attribute::NoCapture) && !Attrs.hasAttribute(Idx, Attribute::Returned) && - !Attrs.hasAttribute(Idx, Attribute::InAlloca), - "Attributes 'byval', 'inalloca', 'nest', 'sret', 'nocapture', and " - "'returned' do not apply to return values!", + !Attrs.hasAttribute(Idx, Attribute::InAlloca) && + !Attrs.hasAttribute(Idx, Attribute::SwiftSelf) && + !Attrs.hasAttribute(Idx, Attribute::SwiftError), + "Attributes 'byval', 'inalloca', 'nest', 'sret', 'nocapture', " + "'returned', 'swiftself', and 'swifterror' do not apply to return " + "values!", V); // Check for mutually incompatible attributes. Only inreg is compatible with @@ -1349,6 +1383,18 @@ void Verifier::VerifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, "'readnone and readonly' are incompatible!", V); + Assert(!(Attrs.hasAttribute(Idx, Attribute::ReadNone) && + Attrs.hasAttribute(Idx, Attribute::WriteOnly)), + "Attributes " + "'readnone and writeonly' are incompatible!", + V); + + Assert(!(Attrs.hasAttribute(Idx, Attribute::ReadOnly) && + Attrs.hasAttribute(Idx, Attribute::WriteOnly)), + "Attributes " + "'readonly and writeonly' are incompatible!", + V); + Assert(!(Attrs.hasAttribute(Idx, Attribute::NoInline) && Attrs.hasAttribute(Idx, Attribute::AlwaysInline)), "Attributes " @@ -1370,16 +1416,25 @@ void Verifier::VerifyParameterAttrs(AttributeSet Attrs, unsigned Idx, Type *Ty, "Attributes 'byval' and 'inalloca' do not support unsized types!", V); } + if (!isa<PointerType>(PTy->getElementType())) + Assert(!Attrs.hasAttribute(Idx, Attribute::SwiftError), + "Attribute 'swifterror' only applies to parameters " + "with pointer to pointer type!", + V); } else { Assert(!Attrs.hasAttribute(Idx, Attribute::ByVal), "Attribute 'byval' only applies to parameters with pointer type!", V); + Assert(!Attrs.hasAttribute(Idx, Attribute::SwiftError), + "Attribute 'swifterror' only applies to parameters " + "with pointer type!", + V); } } -// VerifyFunctionAttrs - Check parameter attributes against a function type. +// Check parameter attributes against a function type. // The value V is printed in error messages. -void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, +void Verifier::verifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, const Value *V) { if (Attrs.isEmpty()) return; @@ -1387,6 +1442,8 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, bool SawNest = false; bool SawReturned = false; bool SawSRet = false; + bool SawSwiftSelf = false; + bool SawSwiftError = false; for (unsigned i = 0, e = Attrs.getNumSlots(); i != e; ++i) { unsigned Idx = Attrs.getSlotIndex(i); @@ -1399,7 +1456,7 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, else break; // VarArgs attributes, verified elsewhere. - VerifyParameterAttrs(Attrs, Idx, Ty, Idx == 0, V); + verifyParameterAttrs(Attrs, Idx, Ty, Idx == 0, V); if (Idx == 0) continue; @@ -1426,6 +1483,17 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, SawSRet = true; } + if (Attrs.hasAttribute(Idx, Attribute::SwiftSelf)) { + Assert(!SawSwiftSelf, "Cannot have multiple 'swiftself' parameters!", V); + SawSwiftSelf = true; + } + + if (Attrs.hasAttribute(Idx, Attribute::SwiftError)) { + Assert(!SawSwiftError, "Cannot have multiple 'swifterror' parameters!", + V); + SawSwiftError = true; + } + if (Attrs.hasAttribute(Idx, Attribute::InAlloca)) { Assert(Idx == FT->getNumParams(), "inalloca isn't on the last parameter!", V); @@ -1435,7 +1503,7 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, if (!Attrs.hasAttributes(AttributeSet::FunctionIndex)) return; - VerifyAttributeTypes(Attrs, AttributeSet::FunctionIndex, true, V); + verifyAttributeTypes(Attrs, AttributeSet::FunctionIndex, true, V); Assert( !(Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::ReadNone) && @@ -1444,6 +1512,16 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, Assert( !(Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::ReadNone) && + Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::WriteOnly)), + "Attributes 'readnone and writeonly' are incompatible!", V); + + Assert( + !(Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::ReadOnly) && + Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::WriteOnly)), + "Attributes 'readonly and writeonly' are incompatible!", V); + + Assert( + !(Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::ReadNone) && Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::InaccessibleMemOrArgMemOnly)), "Attributes 'readnone and inaccessiblemem_or_argmemonly' are incompatible!", V); @@ -1476,19 +1554,43 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT, AttributeSet Attrs, if (Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::JumpTable)) { const GlobalValue *GV = cast<GlobalValue>(V); - Assert(GV->hasUnnamedAddr(), + Assert(GV->hasGlobalUnnamedAddr(), "Attribute 'jumptable' requires 'unnamed_addr'", V); } -} -void Verifier::VerifyFunctionMetadata( - const SmallVector<std::pair<unsigned, MDNode *>, 4> MDs) { - if (MDs.empty()) - return; + if (Attrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::AllocSize)) { + std::pair<unsigned, Optional<unsigned>> Args = + Attrs.getAllocSizeArgs(AttributeSet::FunctionIndex); + + auto CheckParam = [&](StringRef Name, unsigned ParamNo) { + if (ParamNo >= FT->getNumParams()) { + CheckFailed("'allocsize' " + Name + " argument is out of bounds", V); + return false; + } + + if (!FT->getParamType(ParamNo)->isIntegerTy()) { + CheckFailed("'allocsize' " + Name + + " argument must refer to an integer parameter", + V); + return false; + } + + return true; + }; + + if (!CheckParam("element size", Args.first)) + return; - for (unsigned i = 0; i < MDs.size(); i++) { - if (MDs[i].first == LLVMContext::MD_prof) { - MDNode *MD = MDs[i].second; + if (Args.second && !CheckParam("number of elements", *Args.second)) + return; + } +} + +void Verifier::verifyFunctionMetadata( + ArrayRef<std::pair<unsigned, MDNode *>> MDs) { + for (const auto &Pair : MDs) { + if (Pair.first == LLVMContext::MD_prof) { + MDNode *MD = Pair.second; Assert(MD->getNumOperands() == 2, "!prof annotations should have exactly 2 operands", MD); @@ -1525,13 +1627,19 @@ void Verifier::visitConstantExprsRecursively(const Constant *EntryC) { if (const auto *CE = dyn_cast<ConstantExpr>(C)) visitConstantExpr(CE); + if (const auto *GV = dyn_cast<GlobalValue>(C)) { + // Global Values get visited separately, but we do need to make sure + // that the global value is in the correct module + Assert(GV->getParent() == M, "Referencing global in another module!", + EntryC, M, GV, GV->getParent()); + continue; + } + // Visit all sub-expressions. for (const Use &U : C->operands()) { const auto *OpC = dyn_cast<Constant>(U); if (!OpC) continue; - if (isa<GlobalValue>(OpC)) - continue; // Global values get visited separately. if (!ConstantExprVisited.insert(OpC).second) continue; Stack.push_back(OpC); @@ -1548,7 +1656,7 @@ void Verifier::visitConstantExpr(const ConstantExpr *CE) { "Invalid bitcast", CE); } -bool Verifier::VerifyAttributeCount(AttributeSet Attrs, unsigned Params) { +bool Verifier::verifyAttributeCount(AttributeSet Attrs, unsigned Params) { if (Attrs.getNumSlots() == 0) return true; @@ -1562,8 +1670,8 @@ bool Verifier::VerifyAttributeCount(AttributeSet Attrs, unsigned Params) { return false; } -/// \brief Verify that statepoint intrinsic is well formed. -void Verifier::VerifyStatepoint(ImmutableCallSite CS) { +/// Verify that statepoint intrinsic is well formed. +void Verifier::verifyStatepoint(ImmutableCallSite CS) { assert(CS.getCalledFunction() && CS.getCalledFunction()->getIntrinsicID() == Intrinsic::experimental_gc_statepoint); @@ -1674,11 +1782,11 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) { const CallInst *Call = dyn_cast<const CallInst>(U); Assert(Call, "illegal use of statepoint token", &CI, U); if (!Call) continue; - Assert(isa<GCRelocateInst>(Call) || isGCResult(Call), + Assert(isa<GCRelocateInst>(Call) || isa<GCResultInst>(Call), "gc.result or gc.relocate are the only value uses" "of a gc.statepoint", &CI, U); - if (isGCResult(Call)) { + if (isa<GCResultInst>(Call)) { Assert(Call->getArgOperand(0) == &CI, "gc.result connected to wrong gc.statepoint", &CI, Call); } else if (isa<GCRelocateInst>(Call)) { @@ -1766,6 +1874,8 @@ void Verifier::verifySiblingFuncletUnwinds() { // visitFunction - Verify that a function is ok. // void Verifier::visitFunction(const Function &F) { + visitGlobalValue(F); + // Check function arguments. FunctionType *FT = F.getFunctionType(); unsigned NumArgs = F.arg_size(); @@ -1786,11 +1896,11 @@ void Verifier::visitFunction(const Function &F) { AttributeSet Attrs = F.getAttributes(); - Assert(VerifyAttributeCount(Attrs, FT->getNumParams()), + Assert(verifyAttributeCount(Attrs, FT->getNumParams()), "Attribute after last parameter!", &F); // Check function attributes. - VerifyFunctionAttrs(FT, Attrs, &F); + verifyFunctionAttrs(FT, Attrs, &F); // On function declarations/definitions, we do not support the builtin // attribute. We do not check this in VerifyFunctionAttrs since that is @@ -1821,19 +1931,24 @@ void Verifier::visitFunction(const Function &F) { // Check that the argument values match the function type for this function... unsigned i = 0; - for (Function::const_arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; - ++I, ++i) { - Assert(I->getType() == FT->getParamType(i), - "Argument value does not match function argument type!", I, + for (const Argument &Arg : F.args()) { + Assert(Arg.getType() == FT->getParamType(i), + "Argument value does not match function argument type!", &Arg, FT->getParamType(i)); - Assert(I->getType()->isFirstClassType(), - "Function arguments must have first-class types!", I); + Assert(Arg.getType()->isFirstClassType(), + "Function arguments must have first-class types!", &Arg); if (!isLLVMdotName) { - Assert(!I->getType()->isMetadataTy(), - "Function takes metadata but isn't an intrinsic", I, &F); - Assert(!I->getType()->isTokenTy(), - "Function takes token but isn't an intrinsic", I, &F); + Assert(!Arg.getType()->isMetadataTy(), + "Function takes metadata but isn't an intrinsic", &Arg, &F); + Assert(!Arg.getType()->isTokenTy(), + "Function takes token but isn't an intrinsic", &Arg, &F); + } + + // Check that swifterror argument is only used by loads and stores. + if (Attrs.hasAttribute(i+1, Attribute::SwiftError)) { + verifySwiftErrorValue(&Arg); } + ++i; } if (!isLLVMdotName) @@ -1844,7 +1959,7 @@ void Verifier::visitFunction(const Function &F) { SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; F.getAllMetadata(MDs); assert(F.hasMetadata() != MDs.empty() && "Bit out-of-sync"); - VerifyFunctionMetadata(MDs); + verifyFunctionMetadata(MDs); // Check validity of the personality function if (F.hasPersonalityFn()) { @@ -1860,10 +1975,15 @@ void Verifier::visitFunction(const Function &F) { Assert(MDs.empty(), "unmaterialized function cannot have metadata", &F, MDs.empty() ? nullptr : MDs.front().second); } else if (F.isDeclaration()) { - Assert(F.hasExternalLinkage() || F.hasExternalWeakLinkage(), - "invalid linkage type for function declaration", &F); - Assert(MDs.empty(), "function without a body cannot have metadata", &F, - MDs.empty() ? nullptr : MDs.front().second); + for (const auto &I : MDs) { + AssertDI(I.first != LLVMContext::MD_dbg, + "function declaration may not have a !dbg attachment", &F); + Assert(I.first != LLVMContext::MD_prof, + "function declaration may not have a !prof attachment", &F); + + // Verify the metadata itself. + visitMDNode(*I.second); + } Assert(!F.hasPersonalityFn(), "Function declaration shouldn't have a personality routine", &F); } else { @@ -1882,6 +2002,7 @@ void Verifier::visitFunction(const Function &F) { "blockaddress may not be used with the entry block!", Entry); } + unsigned NumDebugAttachments = 0, NumProfAttachments = 0; // Visit metadata attachments. for (const auto &I : MDs) { // Verify that the attachment is legal. @@ -1889,8 +2010,16 @@ void Verifier::visitFunction(const Function &F) { default: break; case LLVMContext::MD_dbg: - Assert(isa<DISubprogram>(I.second), - "function !dbg attachment must be a subprogram", &F, I.second); + ++NumDebugAttachments; + AssertDI(NumDebugAttachments == 1, + "function must have a single !dbg attachment", &F, I.second); + AssertDI(isa<DISubprogram>(I.second), + "function !dbg attachment must be a subprogram", &F, I.second); + break; + case LLVMContext::MD_prof: + ++NumProfAttachments; + Assert(NumProfAttachments == 1, + "function must have a single !prof attachment", &F, I.second); break; } @@ -1918,6 +2047,8 @@ void Verifier::visitFunction(const Function &F) { if (!N) return; + visitDISubprogram(*N); + // Check that all !dbg attachments lead to back to N (or, at least, another // subprogram that describes the same function). // @@ -2053,11 +2184,11 @@ void Verifier::visitSwitchInst(SwitchInst &SI) { // have the same type as the switched-on value. Type *SwitchTy = SI.getCondition()->getType(); SmallPtrSet<ConstantInt*, 32> Constants; - for (SwitchInst::CaseIt i = SI.case_begin(), e = SI.case_end(); i != e; ++i) { - Assert(i.getCaseValue()->getType() == SwitchTy, + for (auto &Case : SI.cases()) { + Assert(Case.getCaseValue()->getType() == SwitchTy, "Switch constants must all be same type as switch value!", &SI); - Assert(Constants.insert(i.getCaseValue()).second, - "Duplicate integer as switch case", &SI, i.getCaseValue()); + Assert(Constants.insert(Case.getCaseValue()).second, + "Duplicate integer as switch case", &SI, Case.getCaseValue()); } visitTerminatorInst(SI); @@ -2362,7 +2493,7 @@ void Verifier::visitPHINode(PHINode &PN) { visitInstruction(PN); } -void Verifier::VerifyCallSite(CallSite CS) { +void Verifier::verifyCallSite(CallSite CS) { Instruction *I = CS.getInstruction(); Assert(CS.getCalledValue()->getType()->isPointerTy(), @@ -2393,11 +2524,11 @@ void Verifier::VerifyCallSite(CallSite CS) { AttributeSet Attrs = CS.getAttributes(); - Assert(VerifyAttributeCount(Attrs, CS.arg_size()), + Assert(verifyAttributeCount(Attrs, CS.arg_size()), "Attribute after last parameter!", I); // Verify call attributes. - VerifyFunctionAttrs(FTy, Attrs, I); + verifyFunctionAttrs(FTy, Attrs, I); // Conservatively check the inalloca argument. // We have a bug if we can find that there is an underlying alloca without @@ -2409,6 +2540,18 @@ void Verifier::VerifyCallSite(CallSite CS) { "inalloca argument for call has mismatched alloca", AI, I); } + // For each argument of the callsite, if it has the swifterror argument, + // make sure the underlying alloca has swifterror as well. + for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) + if (CS.paramHasAttr(i+1, Attribute::SwiftError)) { + Value *SwiftErrorArg = CS.getArgument(i); + auto AI = dyn_cast<AllocaInst>(SwiftErrorArg->stripInBoundsOffsets()); + Assert(AI, "swifterror argument should come from alloca", AI, I); + if (AI) + Assert(AI->isSwiftError(), + "swifterror argument for call has mismatched alloca", AI, I); + } + if (FTy->isVarArg()) { // FIXME? is 'nest' even legal here? bool SawNest = false; @@ -2424,7 +2567,7 @@ void Verifier::VerifyCallSite(CallSite CS) { // Check attributes on the varargs part. for (unsigned Idx = 1 + FTy->getNumParams(); Idx <= CS.arg_size(); ++Idx) { Type *Ty = CS.getArgument(Idx-1)->getType(); - VerifyParameterAttrs(Attrs, Idx, Ty, false, I); + verifyParameterAttrs(Attrs, Idx, Ty, false, I); if (Attrs.hasAttribute(Idx, Attribute::Nest)) { Assert(!SawNest, "More than one parameter has attribute nest!", I); @@ -2469,17 +2612,21 @@ void Verifier::VerifyCallSite(CallSite CS) { if (Intrinsic::ID ID = (Intrinsic::ID)F->getIntrinsicID()) visitIntrinsicCallSite(ID, CS); - // Verify that a callsite has at most one "deopt" and one "funclet" operand - // bundle. - bool FoundDeoptBundle = false, FoundFuncletBundle = false; + // Verify that a callsite has at most one "deopt", at most one "funclet" and + // at most one "gc-transition" operand bundle. + bool FoundDeoptBundle = false, FoundFuncletBundle = false, + FoundGCTransitionBundle = false; for (unsigned i = 0, e = CS.getNumOperandBundles(); i < e; ++i) { OperandBundleUse BU = CS.getOperandBundleAt(i); uint32_t Tag = BU.getTagID(); if (Tag == LLVMContext::OB_deopt) { Assert(!FoundDeoptBundle, "Multiple deopt operand bundles", I); FoundDeoptBundle = true; - } - if (Tag == LLVMContext::OB_funclet) { + } else if (Tag == LLVMContext::OB_gc_transition) { + Assert(!FoundGCTransitionBundle, "Multiple gc-transition operand bundles", + I); + FoundGCTransitionBundle = true; + } else if (Tag == LLVMContext::OB_funclet) { Assert(!FoundFuncletBundle, "Multiple funclet operand bundles", I); FoundFuncletBundle = true; Assert(BU.Inputs.size() == 1, @@ -2490,6 +2637,15 @@ void Verifier::VerifyCallSite(CallSite CS) { } } + // Verify that each inlinable callsite of a debug-info-bearing function in a + // debug-info-bearing function has a debug location attached to it. Failure to + // do so causes assertion failures when the inliner sets up inline scope info. + if (I->getFunction()->getSubprogram() && CS.getCalledFunction() && + CS.getCalledFunction()->getSubprogram()) + Assert(I->getDebugLoc(), "inlinable function call in a function with debug " + "info must have a !dbg location", + I); + visitInstruction(*I); } @@ -2508,7 +2664,8 @@ static bool isTypeCongruent(Type *L, Type *R) { static AttrBuilder getParameterABIAttributes(int I, AttributeSet Attrs) { static const Attribute::AttrKind ABIAttrs[] = { Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, - Attribute::InReg, Attribute::Returned}; + Attribute::InReg, Attribute::Returned, Attribute::SwiftSelf, + Attribute::SwiftError}; AttrBuilder Copy; for (auto AK : ABIAttrs) { if (Attrs.hasAttribute(I + 1, AK)) @@ -2581,14 +2738,14 @@ void Verifier::verifyMustTailCall(CallInst &CI) { } void Verifier::visitCallInst(CallInst &CI) { - VerifyCallSite(&CI); + verifyCallSite(&CI); if (CI.isMustTailCall()) verifyMustTailCall(CI); } void Verifier::visitInvokeInst(InvokeInst &II) { - VerifyCallSite(&II); + verifyCallSite(&II); // Verify that the first non-PHI instruction of the unwind destination is an // exception handling instruction. @@ -2741,8 +2898,8 @@ void Verifier::visitGetElementPtrInst(GetElementPtrInst &GEP) { if (GEP.getPointerOperandType()->isVectorTy()) Assert(GEPWidth == GEP.getPointerOperandType()->getVectorNumElements(), "Vector GEP result width doesn't match operand's", &GEP); - for (unsigned i = 0, e = Idxs.size(); i != e; ++i) { - Type *IndexTy = Idxs[i]->getType(); + for (Value *Idx : Idxs) { + Type *IndexTy = Idx->getType(); if (IndexTy->isVectorTy()) { unsigned IndexWidth = IndexTy->getVectorNumElements(); Assert(IndexWidth == GEPWidth, "Invalid GEP index vector width", &GEP); @@ -2822,8 +2979,10 @@ void Verifier::visitLoadInst(LoadInst &LI) { Type *ElTy = LI.getType(); Assert(LI.getAlignment() <= Value::MaximumAlignment, "huge alignment values are unsupported", &LI); + Assert(ElTy->isSized(), "loading unsized types is not allowed", &LI); if (LI.isAtomic()) { - Assert(LI.getOrdering() != Release && LI.getOrdering() != AcquireRelease, + Assert(LI.getOrdering() != AtomicOrdering::Release && + LI.getOrdering() != AtomicOrdering::AcquireRelease, "Load cannot have Release ordering", &LI); Assert(LI.getAlignment() != 0, "Atomic load must specify explicit alignment", &LI); @@ -2849,8 +3008,10 @@ void Verifier::visitStoreInst(StoreInst &SI) { "Stored value type does not match pointer operand type!", &SI, ElTy); Assert(SI.getAlignment() <= Value::MaximumAlignment, "huge alignment values are unsupported", &SI); + Assert(ElTy->isSized(), "storing unsized types is not allowed", &SI); if (SI.isAtomic()) { - Assert(SI.getOrdering() != Acquire && SI.getOrdering() != AcquireRelease, + Assert(SI.getOrdering() != AtomicOrdering::Acquire && + SI.getOrdering() != AtomicOrdering::AcquireRelease, "Store cannot have Acquire ordering", &SI); Assert(SI.getAlignment() != 0, "Atomic store must specify explicit alignment", &SI); @@ -2867,6 +3028,42 @@ void Verifier::visitStoreInst(StoreInst &SI) { visitInstruction(SI); } +/// Check that SwiftErrorVal is used as a swifterror argument in CS. +void Verifier::verifySwiftErrorCallSite(CallSite CS, + const Value *SwiftErrorVal) { + unsigned Idx = 0; + for (CallSite::arg_iterator I = CS.arg_begin(), E = CS.arg_end(); + I != E; ++I, ++Idx) { + if (*I == SwiftErrorVal) { + Assert(CS.paramHasAttr(Idx+1, Attribute::SwiftError), + "swifterror value when used in a callsite should be marked " + "with swifterror attribute", + SwiftErrorVal, CS); + } + } +} + +void Verifier::verifySwiftErrorValue(const Value *SwiftErrorVal) { + // Check that swifterror value is only used by loads, stores, or as + // a swifterror argument. + for (const User *U : SwiftErrorVal->users()) { + Assert(isa<LoadInst>(U) || isa<StoreInst>(U) || isa<CallInst>(U) || + isa<InvokeInst>(U), + "swifterror value can only be loaded and stored from, or " + "as a swifterror argument!", + SwiftErrorVal, U); + // If it is used by a store, check it is the second operand. + if (auto StoreI = dyn_cast<StoreInst>(U)) + Assert(StoreI->getOperand(1) == SwiftErrorVal, + "swifterror value should be the second operand when used " + "by stores", SwiftErrorVal, U); + if (auto CallI = dyn_cast<CallInst>(U)) + verifySwiftErrorCallSite(const_cast<CallInst*>(CallI), SwiftErrorVal); + if (auto II = dyn_cast<InvokeInst>(U)) + verifySwiftErrorCallSite(const_cast<InvokeInst*>(II), SwiftErrorVal); + } +} + void Verifier::visitAllocaInst(AllocaInst &AI) { SmallPtrSet<Type*, 4> Visited; PointerType *PTy = AI.getType(); @@ -2880,32 +3077,38 @@ void Verifier::visitAllocaInst(AllocaInst &AI) { Assert(AI.getAlignment() <= Value::MaximumAlignment, "huge alignment values are unsupported", &AI); + if (AI.isSwiftError()) { + verifySwiftErrorValue(&AI); + } + visitInstruction(AI); } void Verifier::visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI) { // FIXME: more conditions??? - Assert(CXI.getSuccessOrdering() != NotAtomic, + Assert(CXI.getSuccessOrdering() != AtomicOrdering::NotAtomic, "cmpxchg instructions must be atomic.", &CXI); - Assert(CXI.getFailureOrdering() != NotAtomic, + Assert(CXI.getFailureOrdering() != AtomicOrdering::NotAtomic, "cmpxchg instructions must be atomic.", &CXI); - Assert(CXI.getSuccessOrdering() != Unordered, + Assert(CXI.getSuccessOrdering() != AtomicOrdering::Unordered, "cmpxchg instructions cannot be unordered.", &CXI); - Assert(CXI.getFailureOrdering() != Unordered, + Assert(CXI.getFailureOrdering() != AtomicOrdering::Unordered, "cmpxchg instructions cannot be unordered.", &CXI); - Assert(CXI.getSuccessOrdering() >= CXI.getFailureOrdering(), - "cmpxchg instructions be at least as constrained on success as fail", + Assert(!isStrongerThan(CXI.getFailureOrdering(), CXI.getSuccessOrdering()), + "cmpxchg instructions failure argument shall be no stronger than the " + "success argument", &CXI); - Assert(CXI.getFailureOrdering() != Release && - CXI.getFailureOrdering() != AcquireRelease, + Assert(CXI.getFailureOrdering() != AtomicOrdering::Release && + CXI.getFailureOrdering() != AtomicOrdering::AcquireRelease, "cmpxchg failure ordering cannot include release semantics", &CXI); PointerType *PTy = dyn_cast<PointerType>(CXI.getOperand(0)->getType()); Assert(PTy, "First cmpxchg operand must be a pointer.", &CXI); Type *ElTy = PTy->getElementType(); - Assert(ElTy->isIntegerTy(), "cmpxchg operand must have integer type!", &CXI, - ElTy); + Assert(ElTy->isIntegerTy() || ElTy->isPointerTy(), + "cmpxchg operand must have integer or pointer type", + ElTy, &CXI); checkAtomicMemAccessSize(M, ElTy, &CXI); Assert(ElTy == CXI.getOperand(1)->getType(), "Expected value type does not match pointer operand type!", &CXI, @@ -2916,9 +3119,9 @@ void Verifier::visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI) { } void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) { - Assert(RMWI.getOrdering() != NotAtomic, + Assert(RMWI.getOrdering() != AtomicOrdering::NotAtomic, "atomicrmw instructions must be atomic.", &RMWI); - Assert(RMWI.getOrdering() != Unordered, + Assert(RMWI.getOrdering() != AtomicOrdering::Unordered, "atomicrmw instructions cannot be unordered.", &RMWI); PointerType *PTy = dyn_cast<PointerType>(RMWI.getOperand(0)->getType()); Assert(PTy, "First atomicrmw operand must be a pointer.", &RMWI); @@ -2937,10 +3140,12 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) { void Verifier::visitFenceInst(FenceInst &FI) { const AtomicOrdering Ordering = FI.getOrdering(); - Assert(Ordering == Acquire || Ordering == Release || - Ordering == AcquireRelease || Ordering == SequentiallyConsistent, - "fence instructions may only have " - "acquire, release, acq_rel, or seq_cst ordering.", + Assert(Ordering == AtomicOrdering::Acquire || + Ordering == AtomicOrdering::Release || + Ordering == AtomicOrdering::AcquireRelease || + Ordering == AtomicOrdering::SequentiallyConsistent, + "fence instructions may only have acquire, release, acq_rel, or " + "seq_cst ordering.", &FI); visitInstruction(FI); } @@ -3017,7 +3222,7 @@ void Verifier::visitEHPadPredecessors(Instruction &I) { else FromPad = ConstantTokenNone::get(II->getContext()); } else if (auto *CRI = dyn_cast<CleanupReturnInst>(TI)) { - FromPad = CRI->getCleanupPad(); + FromPad = CRI->getOperand(0); Assert(FromPad != ToPadParent, "A cleanupret must exit its cleanup", CRI); } else if (auto *CSI = dyn_cast<CatchSwitchInst>(TI)) { FromPad = CSI; @@ -3026,6 +3231,7 @@ void Verifier::visitEHPadPredecessors(Instruction &I) { } // The edge may exit from zero or more nested pads. + SmallSet<Value *, 8> Seen; for (;; FromPad = getParentPad(FromPad)) { Assert(FromPad != ToPad, "EH pad cannot handle exceptions raised within it", FromPad, TI); @@ -3035,6 +3241,8 @@ void Verifier::visitEHPadPredecessors(Instruction &I) { } Assert(!isa<ConstantTokenNone>(FromPad), "A single unwind edge may only enter one EH pad", TI); + Assert(Seen.insert(FromPad).second, + "EH pad jumps through a cycle of pads", FromPad); } } } @@ -3081,8 +3289,6 @@ void Verifier::visitLandingPadInst(LandingPadInst &LPI) { } void Verifier::visitCatchPadInst(CatchPadInst &CPI) { - visitEHPadPredecessors(CPI); - BasicBlock *BB = CPI.getParent(); Function *F = BB->getParent(); @@ -3098,6 +3304,7 @@ void Verifier::visitCatchPadInst(CatchPadInst &CPI) { Assert(BB->getFirstNonPHI() == &CPI, "CatchPadInst not the first non-PHI instruction in the block.", &CPI); + visitEHPadPredecessors(CPI); visitFuncletPadInst(CPI); } @@ -3110,8 +3317,6 @@ void Verifier::visitCatchReturnInst(CatchReturnInst &CatchReturn) { } void Verifier::visitCleanupPadInst(CleanupPadInst &CPI) { - visitEHPadPredecessors(CPI); - BasicBlock *BB = CPI.getParent(); Function *F = BB->getParent(); @@ -3128,6 +3333,7 @@ void Verifier::visitCleanupPadInst(CleanupPadInst &CPI) { Assert(isa<ConstantTokenNone>(ParentPad) || isa<FuncletPadInst>(ParentPad), "CleanupPadInst has an invalid parent.", &CPI); + visitEHPadPredecessors(CPI); visitFuncletPadInst(CPI); } @@ -3135,8 +3341,12 @@ void Verifier::visitFuncletPadInst(FuncletPadInst &FPI) { User *FirstUser = nullptr; Value *FirstUnwindPad = nullptr; SmallVector<FuncletPadInst *, 8> Worklist({&FPI}); + SmallSet<FuncletPadInst *, 8> Seen; + while (!Worklist.empty()) { FuncletPadInst *CurrentPad = Worklist.pop_back_val(); + Assert(Seen.insert(CurrentPad).second, + "FuncletPadInst must not be nested within itself", CurrentPad); Value *UnresolvedAncestorPad = nullptr; for (User *U : CurrentPad->users()) { BasicBlock *UnwindDest; @@ -3172,6 +3382,8 @@ void Verifier::visitFuncletPadInst(FuncletPadInst &FPI) { bool ExitsFPI; if (UnwindDest) { UnwindPad = UnwindDest->getFirstNonPHI(); + if (!cast<Instruction>(UnwindPad)->isEHPad()) + continue; Value *UnwindParent = getParentPad(UnwindPad); // Ignore unwind edges that don't exit CurrentPad. if (UnwindParent == CurrentPad) @@ -3285,8 +3497,6 @@ void Verifier::visitFuncletPadInst(FuncletPadInst &FPI) { } void Verifier::visitCatchSwitchInst(CatchSwitchInst &CatchSwitch) { - visitEHPadPredecessors(CatchSwitch); - BasicBlock *BB = CatchSwitch.getParent(); Function *F = BB->getParent(); @@ -3324,6 +3534,7 @@ void Verifier::visitCatchSwitchInst(CatchSwitchInst &CatchSwitch) { "CatchSwitchInst handlers must be catchpads", &CatchSwitch, Handler); } + visitEHPadPredecessors(CatchSwitch); visitTerminatorInst(CatchSwitch); } @@ -3353,8 +3564,18 @@ void Verifier::verifyDominatesUse(Instruction &I, unsigned i) { return; } + // Quick check whether the def has already been encountered in the same block. + // PHI nodes are not checked to prevent accepting preceeding PHIs, because PHI + // uses are defined to happen on the incoming edge, not at the instruction. + // + // FIXME: If this operand is a MetadataAsValue (wrapping a LocalAsMetadata) + // wrapping an SSA value, assert that we've already encountered it. See + // related FIXME in Mapper::mapLocalAsMetadata in ValueMapper.cpp. + if (!isa<PHINode>(I) && InstsInThisBlock.count(Op)) + return; + const Use &U = I.getOperandUse(i); - Assert(InstsInThisBlock.count(Op) || DT.dominates(Op, U), + Assert(DT.dominates(Op, U), "Instruction does not dominate all uses!", Op, &I); } @@ -3435,8 +3656,8 @@ void Verifier::visitInstruction(Instruction &I) { F->getIntrinsicID() == Intrinsic::experimental_patchpoint_void || F->getIntrinsicID() == Intrinsic::experimental_patchpoint_i64 || F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint, - "Cannot invoke an intrinsinc other than" - " donothing or patchpoint", + "Cannot invoke an intrinsic other than donothing, patchpoint or " + "statepoint", &I); Assert(F->getParent() == M, "Referencing function in another module!", &I, M, F, F->getParent()); @@ -3469,7 +3690,9 @@ void Verifier::visitInstruction(Instruction &I) { Assert(MD->getNumOperands() == 1, "fpmath takes one operand!", &I); if (ConstantFP *CFP0 = mdconst::dyn_extract_or_null<ConstantFP>(MD->getOperand(0))) { - APFloat Accuracy = CFP0->getValueAPF(); + const APFloat &Accuracy = CFP0->getValueAPF(); + Assert(&Accuracy.getSemantics() == &APFloat::IEEEsingle, + "fpmath accuracy must have float type", &I); Assert(Accuracy.isFiniteNonZero() && !Accuracy.isNegative(), "fpmath accuracy not a positive number!", &I); } else { @@ -3515,182 +3738,14 @@ void Verifier::visitInstruction(Instruction &I) { } if (MDNode *N = I.getDebugLoc().getAsMDNode()) { - Assert(isa<DILocation>(N), "invalid !dbg metadata attachment", &I, N); + AssertDI(isa<DILocation>(N), "invalid !dbg metadata attachment", &I, N); visitMDNode(*N); } - InstsInThisBlock.insert(&I); -} - -/// VerifyIntrinsicType - Verify that the specified type (which comes from an -/// intrinsic argument or return value) matches the type constraints specified -/// by the .td file (e.g. an "any integer" argument really is an integer). -/// -/// This return true on error but does not print a message. -bool Verifier::VerifyIntrinsicType(Type *Ty, - ArrayRef<Intrinsic::IITDescriptor> &Infos, - SmallVectorImpl<Type*> &ArgTys) { - using namespace Intrinsic; - - // If we ran out of descriptors, there are too many arguments. - if (Infos.empty()) return true; - IITDescriptor D = Infos.front(); - Infos = Infos.slice(1); - - switch (D.Kind) { - case IITDescriptor::Void: return !Ty->isVoidTy(); - case IITDescriptor::VarArg: return true; - case IITDescriptor::MMX: return !Ty->isX86_MMXTy(); - case IITDescriptor::Token: return !Ty->isTokenTy(); - case IITDescriptor::Metadata: return !Ty->isMetadataTy(); - case IITDescriptor::Half: return !Ty->isHalfTy(); - case IITDescriptor::Float: return !Ty->isFloatTy(); - case IITDescriptor::Double: return !Ty->isDoubleTy(); - case IITDescriptor::Integer: return !Ty->isIntegerTy(D.Integer_Width); - case IITDescriptor::Vector: { - VectorType *VT = dyn_cast<VectorType>(Ty); - return !VT || VT->getNumElements() != D.Vector_Width || - VerifyIntrinsicType(VT->getElementType(), Infos, ArgTys); - } - case IITDescriptor::Pointer: { - PointerType *PT = dyn_cast<PointerType>(Ty); - return !PT || PT->getAddressSpace() != D.Pointer_AddressSpace || - VerifyIntrinsicType(PT->getElementType(), Infos, ArgTys); - } - - case IITDescriptor::Struct: { - StructType *ST = dyn_cast<StructType>(Ty); - if (!ST || ST->getNumElements() != D.Struct_NumElements) - return true; - - for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i) - if (VerifyIntrinsicType(ST->getElementType(i), Infos, ArgTys)) - return true; - return false; - } - - case IITDescriptor::Argument: - // Two cases here - If this is the second occurrence of an argument, verify - // that the later instance matches the previous instance. - if (D.getArgumentNumber() < ArgTys.size()) - return Ty != ArgTys[D.getArgumentNumber()]; - - // Otherwise, if this is the first instance of an argument, record it and - // verify the "Any" kind. - assert(D.getArgumentNumber() == ArgTys.size() && "Table consistency error"); - ArgTys.push_back(Ty); - - switch (D.getArgumentKind()) { - case IITDescriptor::AK_Any: return false; // Success - case IITDescriptor::AK_AnyInteger: return !Ty->isIntOrIntVectorTy(); - case IITDescriptor::AK_AnyFloat: return !Ty->isFPOrFPVectorTy(); - case IITDescriptor::AK_AnyVector: return !isa<VectorType>(Ty); - case IITDescriptor::AK_AnyPointer: return !isa<PointerType>(Ty); - } - llvm_unreachable("all argument kinds not covered"); - - case IITDescriptor::ExtendArgument: { - // This may only be used when referring to a previous vector argument. - if (D.getArgumentNumber() >= ArgTys.size()) - return true; - - Type *NewTy = ArgTys[D.getArgumentNumber()]; - if (VectorType *VTy = dyn_cast<VectorType>(NewTy)) - NewTy = VectorType::getExtendedElementVectorType(VTy); - else if (IntegerType *ITy = dyn_cast<IntegerType>(NewTy)) - NewTy = IntegerType::get(ITy->getContext(), 2 * ITy->getBitWidth()); - else - return true; - - return Ty != NewTy; - } - case IITDescriptor::TruncArgument: { - // This may only be used when referring to a previous vector argument. - if (D.getArgumentNumber() >= ArgTys.size()) - return true; - - Type *NewTy = ArgTys[D.getArgumentNumber()]; - if (VectorType *VTy = dyn_cast<VectorType>(NewTy)) - NewTy = VectorType::getTruncatedElementVectorType(VTy); - else if (IntegerType *ITy = dyn_cast<IntegerType>(NewTy)) - NewTy = IntegerType::get(ITy->getContext(), ITy->getBitWidth() / 2); - else - return true; - - return Ty != NewTy; - } - case IITDescriptor::HalfVecArgument: - // This may only be used when referring to a previous vector argument. - return D.getArgumentNumber() >= ArgTys.size() || - !isa<VectorType>(ArgTys[D.getArgumentNumber()]) || - VectorType::getHalfElementsVectorType( - cast<VectorType>(ArgTys[D.getArgumentNumber()])) != Ty; - case IITDescriptor::SameVecWidthArgument: { - if (D.getArgumentNumber() >= ArgTys.size()) - return true; - VectorType * ReferenceType = - dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]); - VectorType *ThisArgType = dyn_cast<VectorType>(Ty); - if (!ThisArgType || !ReferenceType || - (ReferenceType->getVectorNumElements() != - ThisArgType->getVectorNumElements())) - return true; - return VerifyIntrinsicType(ThisArgType->getVectorElementType(), - Infos, ArgTys); - } - case IITDescriptor::PtrToArgument: { - if (D.getArgumentNumber() >= ArgTys.size()) - return true; - Type * ReferenceType = ArgTys[D.getArgumentNumber()]; - PointerType *ThisArgType = dyn_cast<PointerType>(Ty); - return (!ThisArgType || ThisArgType->getElementType() != ReferenceType); - } - case IITDescriptor::VecOfPtrsToElt: { - if (D.getArgumentNumber() >= ArgTys.size()) - return true; - VectorType * ReferenceType = - dyn_cast<VectorType> (ArgTys[D.getArgumentNumber()]); - VectorType *ThisArgVecTy = dyn_cast<VectorType>(Ty); - if (!ThisArgVecTy || !ReferenceType || - (ReferenceType->getVectorNumElements() != - ThisArgVecTy->getVectorNumElements())) - return true; - PointerType *ThisArgEltTy = - dyn_cast<PointerType>(ThisArgVecTy->getVectorElementType()); - if (!ThisArgEltTy) - return true; - return ThisArgEltTy->getElementType() != - ReferenceType->getVectorElementType(); - } - } - llvm_unreachable("unhandled"); -} - -/// \brief Verify if the intrinsic has variable arguments. -/// This method is intended to be called after all the fixed arguments have been -/// verified first. -/// -/// This method returns true on error and does not print an error message. -bool -Verifier::VerifyIntrinsicIsVarArg(bool isVarArg, - ArrayRef<Intrinsic::IITDescriptor> &Infos) { - using namespace Intrinsic; - - // If there are no descriptors left, then it can't be a vararg. - if (Infos.empty()) - return isVarArg; - - // There should be only one descriptor remaining at this point. - if (Infos.size() != 1) - return true; - - // Check and verify the descriptor. - IITDescriptor D = Infos.front(); - Infos = Infos.slice(1); - if (D.Kind == IITDescriptor::VarArg) - return !isVarArg; + if (auto *DII = dyn_cast<DbgInfoIntrinsic>(&I)) + verifyBitPieceExpression(*DII); - return true; + InstsInThisBlock.insert(&I); } /// Allow intrinsics to be verified in different ways. @@ -3709,18 +3764,20 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { ArrayRef<Intrinsic::IITDescriptor> TableRef = Table; SmallVector<Type *, 4> ArgTys; - Assert(!VerifyIntrinsicType(IFTy->getReturnType(), TableRef, ArgTys), + Assert(!Intrinsic::matchIntrinsicType(IFTy->getReturnType(), + TableRef, ArgTys), "Intrinsic has incorrect return type!", IF); for (unsigned i = 0, e = IFTy->getNumParams(); i != e; ++i) - Assert(!VerifyIntrinsicType(IFTy->getParamType(i), TableRef, ArgTys), + Assert(!Intrinsic::matchIntrinsicType(IFTy->getParamType(i), + TableRef, ArgTys), "Intrinsic has incorrect argument type!", IF); // Verify if the intrinsic call matches the vararg property. if (IsVarArg) - Assert(!VerifyIntrinsicIsVarArg(IsVarArg, TableRef), + Assert(!Intrinsic::matchIntrinsicVarArg(IsVarArg, TableRef), "Intrinsic was not defined with variable arguments!", IF); else - Assert(!VerifyIntrinsicIsVarArg(IsVarArg, TableRef), + Assert(!Intrinsic::matchIntrinsicVarArg(IsVarArg, TableRef), "Callsite was not defined with variable arguments!", IF); // All descriptors should be absorbed by now. @@ -3863,7 +3920,7 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { Assert(CS.getParent()->getParent()->hasGC(), "Enclosing function does not use GC.", CS); - VerifyStatepoint(CS); + verifyStatepoint(CS); break; case Intrinsic::experimental_gc_result: { Assert(CS.getParent()->getParent()->hasGC(), @@ -3913,18 +3970,18 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { else { // In all other cases relocate should be tied to the statepoint directly. // This covers relocates on a normal return path of invoke statepoint and - // relocates of a call statepoint + // relocates of a call statepoint. auto Token = CS.getArgOperand(0); Assert(isa<Instruction>(Token) && isStatepoint(cast<Instruction>(Token)), "gc relocate is incorrectly tied to the statepoint", CS, Token); } - // Verify rest of the relocate arguments + // Verify rest of the relocate arguments. ImmutableCallSite StatepointCS( cast<GCRelocateInst>(*CS.getInstruction()).getStatepoint()); - // Both the base and derived must be piped through the safepoint + // Both the base and derived must be piped through the safepoint. Value* Base = CS.getArgOperand(1); Assert(isa<ConstantInt>(Base), "gc.relocate operand #2 must be integer offset", CS); @@ -3942,7 +3999,7 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { "gc.relocate: statepoint derived index out of bounds", CS); // Check that BaseIndex and DerivedIndex fall within the 'gc parameters' - // section of the statepoint's argument + // section of the statepoint's argument. Assert(StatepointCS.arg_size() > 0, "gc.statepoint: insufficient arguments"); Assert(isa<ConstantInt>(StatepointCS.getArgument(3)), @@ -3962,7 +4019,8 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { "gc.statepoint: number of deoptimization arguments must be " "a constant integer"); const int NumDeoptArgs = - cast<ConstantInt>(StatepointCS.getArgument(DeoptArgsStart))->getZExtValue(); + cast<ConstantInt>(StatepointCS.getArgument(DeoptArgsStart)) + ->getZExtValue(); const int GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs; const int GCParamArgsEnd = StatepointCS.arg_size(); Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd, @@ -3985,10 +4043,13 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { auto ResultType = CS.getType(); auto DerivedType = Relocate.getDerivedPtr()->getType(); Assert(ResultType->isVectorTy() == DerivedType->isVectorTy(), - "gc.relocate: vector relocates to vector and pointer to pointer", CS); - Assert(ResultType->getPointerAddressSpace() == - DerivedType->getPointerAddressSpace(), - "gc.relocate: relocating a pointer shouldn't change its address space", CS); + "gc.relocate: vector relocates to vector and pointer to pointer", + CS); + Assert( + ResultType->getPointerAddressSpace() == + DerivedType->getPointerAddressSpace(), + "gc.relocate: relocating a pointer shouldn't change its address space", + CS); break; } case Intrinsic::eh_exceptioncode: @@ -3997,6 +4058,75 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { "eh.exceptionpointer argument must be a catchpad", CS); break; } + case Intrinsic::masked_load: { + Assert(CS.getType()->isVectorTy(), "masked_load: must return a vector", CS); + + Value *Ptr = CS.getArgOperand(0); + //Value *Alignment = CS.getArgOperand(1); + Value *Mask = CS.getArgOperand(2); + Value *PassThru = CS.getArgOperand(3); + Assert(Mask->getType()->isVectorTy(), + "masked_load: mask must be vector", CS); + + // DataTy is the overloaded type + Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType(); + Assert(DataTy == CS.getType(), + "masked_load: return must match pointer type", CS); + Assert(PassThru->getType() == DataTy, + "masked_load: pass through and data type must match", CS); + Assert(Mask->getType()->getVectorNumElements() == + DataTy->getVectorNumElements(), + "masked_load: vector mask must be same length as data", CS); + break; + } + case Intrinsic::masked_store: { + Value *Val = CS.getArgOperand(0); + Value *Ptr = CS.getArgOperand(1); + //Value *Alignment = CS.getArgOperand(2); + Value *Mask = CS.getArgOperand(3); + Assert(Mask->getType()->isVectorTy(), + "masked_store: mask must be vector", CS); + + // DataTy is the overloaded type + Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType(); + Assert(DataTy == Val->getType(), + "masked_store: storee must match pointer type", CS); + Assert(Mask->getType()->getVectorNumElements() == + DataTy->getVectorNumElements(), + "masked_store: vector mask must be same length as data", CS); + break; + } + + case Intrinsic::experimental_guard: { + Assert(CS.isCall(), "experimental_guard cannot be invoked", CS); + Assert(CS.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1, + "experimental_guard must have exactly one " + "\"deopt\" operand bundle"); + break; + } + + case Intrinsic::experimental_deoptimize: { + Assert(CS.isCall(), "experimental_deoptimize cannot be invoked", CS); + Assert(CS.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1, + "experimental_deoptimize must have exactly one " + "\"deopt\" operand bundle"); + Assert(CS.getType() == CS.getInstruction()->getFunction()->getReturnType(), + "experimental_deoptimize return type must match caller return type"); + + if (CS.isCall()) { + auto *DeoptCI = CS.getInstruction(); + auto *RI = dyn_cast<ReturnInst>(DeoptCI->getNextNode()); + Assert(RI, + "calls to experimental_deoptimize must be followed by a return"); + + if (!CS.getType()->isVoidTy() && RI) + Assert(RI->getReturnValue() == DeoptCI, + "calls to experimental_deoptimize must be followed by a return " + "of the value computed by experimental_deoptimize"); + } + + break; + } }; } @@ -4022,13 +4152,13 @@ static DISubprogram *getSubprogram(Metadata *LocalScope) { template <class DbgIntrinsicTy> void Verifier::visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII) { auto *MD = cast<MetadataAsValue>(DII.getArgOperand(0))->getMetadata(); - Assert(isa<ValueAsMetadata>(MD) || + AssertDI(isa<ValueAsMetadata>(MD) || (isa<MDNode>(MD) && !cast<MDNode>(MD)->getNumOperands()), "invalid llvm.dbg." + Kind + " intrinsic address/value", &DII, MD); - Assert(isa<DILocalVariable>(DII.getRawVariable()), + AssertDI(isa<DILocalVariable>(DII.getRawVariable()), "invalid llvm.dbg." + Kind + " intrinsic variable", &DII, DII.getRawVariable()); - Assert(isa<DIExpression>(DII.getRawExpression()), + AssertDI(isa<DIExpression>(DII.getRawExpression()), "invalid llvm.dbg." + Kind + " intrinsic expression", &DII, DII.getRawExpression()); @@ -4057,8 +4187,7 @@ void Verifier::visitDbgIntrinsic(StringRef Kind, DbgIntrinsicTy &DII) { Loc->getScope()->getSubprogram()); } -template <class MapTy> -static uint64_t getVariableSize(const DILocalVariable &V, const MapTy &Map) { +static uint64_t getVariableSize(const DILocalVariable &V) { // Be careful of broken types (checked elsewhere). const Metadata *RawType = V.getRawType(); while (RawType) { @@ -4073,12 +4202,6 @@ static uint64_t getVariableSize(const DILocalVariable &V, const MapTy &Map) { continue; } - if (auto *S = dyn_cast<MDString>(RawType)) { - // Don't error on missing types (checked elsewhere). - RawType = Map.lookup(S); - continue; - } - // Missing type or size. break; } @@ -4087,9 +4210,7 @@ static uint64_t getVariableSize(const DILocalVariable &V, const MapTy &Map) { return 0; } -template <class MapTy> -void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I, - const MapTy &TypeRefs) { +void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I) { DILocalVariable *V; DIExpression *E; if (auto *DVI = dyn_cast<DbgValueInst>(&I)) { @@ -4120,7 +4241,7 @@ void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I, // If there's no size, the type is broken, but that should be checked // elsewhere. - uint64_t VarSize = getVariableSize(*V, TypeRefs); + uint64_t VarSize = getVariableSize(*V); if (!VarSize) return; @@ -4131,54 +4252,29 @@ void Verifier::verifyBitPieceExpression(const DbgInfoIntrinsic &I, Assert(PieceSize != VarSize, "piece covers entire variable", &I, V, E); } -void Verifier::visitUnresolvedTypeRef(const MDString *S, const MDNode *N) { - // This is in its own function so we get an error for each bad type ref (not - // just the first). - Assert(false, "unresolved type ref", S, N); -} - -void Verifier::verifyTypeRefs() { +void Verifier::verifyCompileUnits() { auto *CUs = M->getNamedMetadata("llvm.dbg.cu"); - if (!CUs) - return; - - // Visit all the compile units again to map the type references. - SmallDenseMap<const MDString *, const DIType *, 32> TypeRefs; - for (auto *CU : CUs->operands()) - if (auto Ts = cast<DICompileUnit>(CU)->getRetainedTypes()) - for (DIType *Op : Ts) - if (auto *T = dyn_cast_or_null<DICompositeType>(Op)) - if (auto *S = T->getRawIdentifier()) { - UnresolvedTypeRefs.erase(S); - TypeRefs.insert(std::make_pair(S, T)); - } + SmallPtrSet<const Metadata *, 2> Listed; + if (CUs) + Listed.insert(CUs->op_begin(), CUs->op_end()); + Assert( + std::all_of(CUVisited.begin(), CUVisited.end(), + [&Listed](const Metadata *CU) { return Listed.count(CU); }), + "All DICompileUnits must be listed in llvm.dbg.cu"); + CUVisited.clear(); +} - // Verify debug info intrinsic bit piece expressions. This needs a second - // pass through the intructions, since we haven't built TypeRefs yet when - // verifying functions, and simply queuing the DbgInfoIntrinsics to evaluate - // later/now would queue up some that could be later deleted. - for (const Function &F : *M) - for (const BasicBlock &BB : F) - for (const Instruction &I : BB) - if (auto *DII = dyn_cast<DbgInfoIntrinsic>(&I)) - verifyBitPieceExpression(*DII, TypeRefs); - - // Return early if all typerefs were resolved. - if (UnresolvedTypeRefs.empty()) +void Verifier::verifyDeoptimizeCallingConvs() { + if (DeoptimizeDeclarations.empty()) return; - // Sort the unresolved references by name so the output is deterministic. - typedef std::pair<const MDString *, const MDNode *> TypeRef; - SmallVector<TypeRef, 32> Unresolved(UnresolvedTypeRefs.begin(), - UnresolvedTypeRefs.end()); - std::sort(Unresolved.begin(), Unresolved.end(), - [](const TypeRef &LHS, const TypeRef &RHS) { - return LHS.first->getString() < RHS.first->getString(); - }); - - // Visit the unresolved refs (printing out the errors). - for (const TypeRef &TR : Unresolved) - visitUnresolvedTypeRef(TR.first, TR.second); + const Function *First = DeoptimizeDeclarations[0]; + for (auto *F : makeArrayRef(DeoptimizeDeclarations).slice(1)) { + Assert(First->getCallingConv() == F->getCallingConv(), + "All llvm.experimental.deoptimize declarations must have the same " + "calling convention", + First, F); + } } //===----------------------------------------------------------------------===// @@ -4187,28 +4283,30 @@ void Verifier::verifyTypeRefs() { bool llvm::verifyFunction(const Function &f, raw_ostream *OS) { Function &F = const_cast<Function &>(f); - assert(!F.isDeclaration() && "Cannot verify external functions"); - raw_null_ostream NullStr; - Verifier V(OS ? *OS : NullStr); + // Don't use a raw_null_ostream. Printing IR is expensive. + Verifier V(OS, /*ShouldTreatBrokenDebugInfoAsError=*/true); // Note that this function's return value is inverted from what you would // expect of a function called "verify". return !V.verify(F); } -bool llvm::verifyModule(const Module &M, raw_ostream *OS) { - raw_null_ostream NullStr; - Verifier V(OS ? *OS : NullStr); +bool llvm::verifyModule(const Module &M, raw_ostream *OS, + bool *BrokenDebugInfo) { + // Don't use a raw_null_ostream. Printing IR is expensive. + Verifier V(OS, /*ShouldTreatBrokenDebugInfoAsError=*/!BrokenDebugInfo); bool Broken = false; - for (Module::const_iterator I = M.begin(), E = M.end(); I != E; ++I) - if (!I->isDeclaration() && !I->isMaterializable()) - Broken |= !V.verify(*I); + for (const Function &F : M) + Broken |= !V.verify(F); + Broken |= !V.verify(M); + if (BrokenDebugInfo) + *BrokenDebugInfo = V.hasBrokenDebugInfo(); // Note that this function's return value is inverted from what you would // expect of a function called "verify". - return !V.verify(M) || Broken; + return Broken; } namespace { @@ -4216,13 +4314,17 @@ struct VerifierLegacyPass : public FunctionPass { static char ID; Verifier V; - bool FatalErrors; + bool FatalErrors = true; - VerifierLegacyPass() : FunctionPass(ID), V(dbgs()), FatalErrors(true) { + VerifierLegacyPass() + : FunctionPass(ID), + V(&dbgs(), /*ShouldTreatBrokenDebugInfoAsError=*/false) { initializeVerifierLegacyPassPass(*PassRegistry::getPassRegistry()); } explicit VerifierLegacyPass(bool FatalErrors) - : FunctionPass(ID), V(dbgs()), FatalErrors(FatalErrors) { + : FunctionPass(ID), + V(&dbgs(), /*ShouldTreatBrokenDebugInfoAsError=*/false), + FatalErrors(FatalErrors) { initializeVerifierLegacyPassPass(*PassRegistry::getPassRegistry()); } @@ -4234,9 +4336,25 @@ struct VerifierLegacyPass : public FunctionPass { } bool doFinalization(Module &M) override { - if (!V.verify(M) && FatalErrors) - report_fatal_error("Broken module found, compilation aborted!"); + bool HasErrors = false; + for (Function &F : M) + if (F.isDeclaration()) + HasErrors |= !V.verify(F); + + HasErrors |= !V.verify(M); + if (FatalErrors) { + if (HasErrors) + report_fatal_error("Broken module found, compilation aborted!"); + assert(!V.hasBrokenDebugInfo() && "Module contains invalid debug info"); + } + // Strip broken debug info. + if (V.hasBrokenDebugInfo()) { + DiagnosticInfoIgnoringInvalidDebugMetadata DiagInvalid(M); + M.getContext().diagnose(DiagInvalid); + if (!StripDebugInfo(M)) + report_fatal_error("Failed to strip malformed debug info"); + } return false; } @@ -4253,15 +4371,40 @@ FunctionPass *llvm::createVerifierPass(bool FatalErrors) { return new VerifierLegacyPass(FatalErrors); } -PreservedAnalyses VerifierPass::run(Module &M) { - if (verifyModule(M, &dbgs()) && FatalErrors) - report_fatal_error("Broken module found, compilation aborted!"); +char VerifierAnalysis::PassID; +VerifierAnalysis::Result VerifierAnalysis::run(Module &M, + ModuleAnalysisManager &) { + Result Res; + Res.IRBroken = llvm::verifyModule(M, &dbgs(), &Res.DebugInfoBroken); + return Res; +} + +VerifierAnalysis::Result VerifierAnalysis::run(Function &F, + FunctionAnalysisManager &) { + return { llvm::verifyFunction(F, &dbgs()), false }; +} +PreservedAnalyses VerifierPass::run(Module &M, ModuleAnalysisManager &AM) { + auto Res = AM.getResult<VerifierAnalysis>(M); + if (FatalErrors) { + if (Res.IRBroken) + report_fatal_error("Broken module found, compilation aborted!"); + assert(!Res.DebugInfoBroken && "Module contains invalid debug info"); + } + + // Strip broken debug info. + if (Res.DebugInfoBroken) { + DiagnosticInfoIgnoringInvalidDebugMetadata DiagInvalid(M); + M.getContext().diagnose(DiagInvalid); + if (!StripDebugInfo(M)) + report_fatal_error("Failed to strip malformed debug info"); + } return PreservedAnalyses::all(); } -PreservedAnalyses VerifierPass::run(Function &F) { - if (verifyFunction(F, &dbgs()) && FatalErrors) +PreservedAnalyses VerifierPass::run(Function &F, FunctionAnalysisManager &AM) { + auto res = AM.getResult<VerifierAnalysis>(F); + if (res.IRBroken && FatalErrors) report_fatal_error("Broken function found, compilation aborted!"); return PreservedAnalyses::all(); diff --git a/lib/IR/module.modulemap b/lib/IR/module.modulemap deleted file mode 100644 index 9698e91783496..0000000000000 --- a/lib/IR/module.modulemap +++ /dev/null @@ -1 +0,0 @@ -module IR { requires cplusplus umbrella "." module * { export * } } |